b4c5a13664
As seen on #1174809, changes use of mutable types as default arguments and defaults them within the method. Otherwise, those defaults can be unexpectedly persisted with the function between invocations and erupt into mass hysteria on the streets. There was indeed a test (TestSimpleClient.test_get_with_retries) that was erroneously relying on this behavior. Since previous tests had populated their own instantiations with a token, this test only passed because the modified headers dict from previous tests was being overridden. As expected, with the mutable defaults fix in SimpleClient, this test begain to fail since it never specified any token, yet it has always passed anyway. This change also now provides the expected token. Change-Id: If95f11d259008517dab511e88acfe9731e5a99b5 Related-Bug: #1174809
953 lines
30 KiB
Python
953 lines
30 KiB
Python
# Copyright (c) 2010-2012 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import hashlib
|
|
import httplib
|
|
import os
|
|
import random
|
|
import socket
|
|
import StringIO
|
|
import time
|
|
import urllib
|
|
|
|
import simplejson as json
|
|
|
|
from nose import SkipTest
|
|
from xml.dom import minidom
|
|
from swiftclient import get_auth
|
|
|
|
from test import safe_repr
|
|
|
|
|
|
class AuthenticationFailed(Exception):
|
|
pass
|
|
|
|
|
|
class RequestError(Exception):
|
|
pass
|
|
|
|
|
|
class ResponseError(Exception):
|
|
def __init__(self, response, method=None, path=None):
|
|
self.status = response.status
|
|
self.reason = response.reason
|
|
self.method = method
|
|
self.path = path
|
|
self.headers = response.getheaders()
|
|
|
|
for name, value in self.headers:
|
|
if name.lower() == 'x-trans-id':
|
|
self.txid = value
|
|
break
|
|
else:
|
|
self.txid = None
|
|
|
|
super(ResponseError, self).__init__()
|
|
|
|
def __str__(self):
|
|
return repr(self)
|
|
|
|
def __repr__(self):
|
|
return '%d: %r (%r %r) txid=%s' % (
|
|
self.status, self.reason, self.method, self.path, self.txid)
|
|
|
|
|
|
def listing_empty(method):
|
|
for i in xrange(6):
|
|
if len(method()) == 0:
|
|
return True
|
|
|
|
time.sleep(2 ** i)
|
|
|
|
return False
|
|
|
|
|
|
def listing_items(method):
|
|
marker = None
|
|
once = True
|
|
items = []
|
|
|
|
while once or items:
|
|
for i in items:
|
|
yield i
|
|
|
|
if once or marker:
|
|
if marker:
|
|
items = method(parms={'marker': marker})
|
|
else:
|
|
items = method()
|
|
|
|
if len(items) == 10000:
|
|
marker = items[-1]
|
|
else:
|
|
marker = None
|
|
|
|
once = False
|
|
else:
|
|
items = []
|
|
|
|
|
|
class Connection(object):
|
|
def __init__(self, config):
|
|
for key in 'auth_host auth_port auth_ssl username password'.split():
|
|
if key not in config:
|
|
raise SkipTest(
|
|
"Missing required configuration parameter: %s" % key)
|
|
|
|
self.auth_host = config['auth_host']
|
|
self.auth_port = int(config['auth_port'])
|
|
self.auth_ssl = config['auth_ssl'] in ('on', 'true', 'yes', '1')
|
|
self.auth_prefix = config.get('auth_prefix', '/')
|
|
self.auth_version = str(config.get('auth_version', '1'))
|
|
|
|
self.account = config.get('account')
|
|
self.username = config['username']
|
|
self.password = config['password']
|
|
|
|
self.storage_host = None
|
|
self.storage_port = None
|
|
self.storage_url = None
|
|
|
|
self.conn_class = None
|
|
|
|
def get_account(self):
|
|
return Account(self, self.account)
|
|
|
|
def authenticate(self, clone_conn=None):
|
|
if clone_conn:
|
|
self.conn_class = clone_conn.conn_class
|
|
self.storage_host = clone_conn.storage_host
|
|
self.storage_url = clone_conn.storage_url
|
|
self.storage_port = clone_conn.storage_port
|
|
self.storage_token = clone_conn.storage_token
|
|
return
|
|
|
|
if self.auth_version == "1":
|
|
auth_path = '%sv1.0' % (self.auth_prefix)
|
|
if self.account:
|
|
auth_user = '%s:%s' % (self.account, self.username)
|
|
else:
|
|
auth_user = self.username
|
|
else:
|
|
auth_user = self.username
|
|
auth_path = self.auth_prefix
|
|
auth_scheme = 'https://' if self.auth_ssl else 'http://'
|
|
auth_netloc = "%s:%d" % (self.auth_host, self.auth_port)
|
|
auth_url = auth_scheme + auth_netloc + auth_path
|
|
|
|
(storage_url, storage_token) = get_auth(
|
|
auth_url, auth_user, self.password, snet=False,
|
|
tenant_name=self.account, auth_version=self.auth_version,
|
|
os_options={})
|
|
|
|
if not (storage_url and storage_token):
|
|
raise AuthenticationFailed()
|
|
|
|
x = storage_url.split('/')
|
|
|
|
if x[0] == 'http:':
|
|
self.conn_class = httplib.HTTPConnection
|
|
self.storage_port = 80
|
|
elif x[0] == 'https:':
|
|
self.conn_class = httplib.HTTPSConnection
|
|
self.storage_port = 443
|
|
else:
|
|
raise ValueError('unexpected protocol %s' % (x[0]))
|
|
|
|
self.storage_host = x[2].split(':')[0]
|
|
if ':' in x[2]:
|
|
self.storage_port = int(x[2].split(':')[1])
|
|
# Make sure storage_url is a string and not unicode, since
|
|
# keystoneclient (called by swiftclient) returns them in
|
|
# unicode and this would cause troubles when doing
|
|
# no_safe_quote query.
|
|
self.storage_url = str('/%s/%s' % (x[3], x[4]))
|
|
|
|
self.storage_token = storage_token
|
|
|
|
self.http_connect()
|
|
return self.storage_url, self.storage_token
|
|
|
|
def cluster_info(self):
|
|
"""
|
|
Retrieve the data in /info, or {} on 404
|
|
"""
|
|
status = self.make_request('GET', '/info',
|
|
cfg={'absolute_path': True})
|
|
if status == 404:
|
|
return {}
|
|
if not 200 <= status <= 299:
|
|
raise ResponseError(self.response, 'GET', '/info')
|
|
return json.loads(self.response.read())
|
|
|
|
def http_connect(self):
|
|
self.connection = self.conn_class(self.storage_host,
|
|
port=self.storage_port)
|
|
#self.connection.set_debuglevel(3)
|
|
|
|
def make_path(self, path=None, cfg=None):
|
|
if path is None:
|
|
path = []
|
|
if cfg is None:
|
|
cfg = {}
|
|
|
|
if cfg.get('version_only_path'):
|
|
return '/' + self.storage_url.split('/')[1]
|
|
|
|
if path:
|
|
quote = urllib.quote
|
|
if cfg.get('no_quote') or cfg.get('no_path_quote'):
|
|
quote = lambda x: x
|
|
return '%s/%s' % (self.storage_url,
|
|
'/'.join([quote(i) for i in path]))
|
|
else:
|
|
return self.storage_url
|
|
|
|
def make_headers(self, hdrs, cfg=None):
|
|
if cfg is None:
|
|
cfg = {}
|
|
headers = {}
|
|
|
|
if not cfg.get('no_auth_token'):
|
|
headers['X-Auth-Token'] = self.storage_token
|
|
|
|
if isinstance(hdrs, dict):
|
|
headers.update(hdrs)
|
|
return headers
|
|
|
|
def make_request(self, method, path=None, data='', hdrs=None, parms=None,
|
|
cfg=None):
|
|
if path is None:
|
|
path = []
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
if not cfg.get('absolute_path'):
|
|
# Set absolute_path=True to make a request to exactly the given
|
|
# path, not storage path + given path. Useful for
|
|
# non-account/container/object requests.
|
|
path = self.make_path(path, cfg=cfg)
|
|
headers = self.make_headers(hdrs, cfg=cfg)
|
|
if isinstance(parms, dict) and parms:
|
|
quote = urllib.quote
|
|
if cfg.get('no_quote') or cfg.get('no_parms_quote'):
|
|
quote = lambda x: x
|
|
query_args = ['%s=%s' % (quote(x), quote(str(y)))
|
|
for (x, y) in parms.items()]
|
|
path = '%s?%s' % (path, '&'.join(query_args))
|
|
if not cfg.get('no_content_length'):
|
|
if cfg.get('set_content_length'):
|
|
headers['Content-Length'] = cfg.get('set_content_length')
|
|
else:
|
|
headers['Content-Length'] = len(data)
|
|
|
|
def try_request():
|
|
self.http_connect()
|
|
self.connection.request(method, path, data, headers)
|
|
return self.connection.getresponse()
|
|
|
|
self.response = None
|
|
try_count = 0
|
|
fail_messages = []
|
|
while try_count < 5:
|
|
try_count += 1
|
|
|
|
try:
|
|
self.response = try_request()
|
|
except httplib.HTTPException as e:
|
|
fail_messages.append(safe_repr(e))
|
|
continue
|
|
|
|
if self.response.status == 401:
|
|
fail_messages.append("Response 401")
|
|
self.authenticate()
|
|
continue
|
|
elif self.response.status == 503:
|
|
fail_messages.append("Response 503")
|
|
if try_count != 5:
|
|
time.sleep(5)
|
|
continue
|
|
|
|
break
|
|
|
|
if self.response:
|
|
return self.response.status
|
|
|
|
request = "{method} {path} headers: {headers} data: {data}".format(
|
|
method=method, path=path, headers=headers, data=data)
|
|
raise RequestError('Unable to complete http request: %s. '
|
|
'Attempts: %s, Failures: %s' %
|
|
(request, len(fail_messages), fail_messages))
|
|
|
|
def put_start(self, path, hdrs=None, parms=None, cfg=None, chunked=False):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
|
|
self.http_connect()
|
|
|
|
path = self.make_path(path, cfg)
|
|
headers = self.make_headers(hdrs, cfg=cfg)
|
|
|
|
if chunked:
|
|
headers['Transfer-Encoding'] = 'chunked'
|
|
headers.pop('Content-Length', None)
|
|
|
|
if isinstance(parms, dict) and parms:
|
|
quote = urllib.quote
|
|
if cfg.get('no_quote') or cfg.get('no_parms_quote'):
|
|
quote = lambda x: x
|
|
query_args = ['%s=%s' % (quote(x), quote(str(y)))
|
|
for (x, y) in parms.items()]
|
|
path = '%s?%s' % (path, '&'.join(query_args))
|
|
|
|
self.connection = self.conn_class(self.storage_host,
|
|
port=self.storage_port)
|
|
#self.connection.set_debuglevel(3)
|
|
self.connection.putrequest('PUT', path)
|
|
for key, value in headers.iteritems():
|
|
self.connection.putheader(key, value)
|
|
self.connection.endheaders()
|
|
|
|
def put_data(self, data, chunked=False):
|
|
if chunked:
|
|
self.connection.send('%x\r\n%s\r\n' % (len(data), data))
|
|
else:
|
|
self.connection.send(data)
|
|
|
|
def put_end(self, chunked=False):
|
|
if chunked:
|
|
self.connection.send('0\r\n\r\n')
|
|
|
|
self.response = self.connection.getresponse()
|
|
self.connection.close()
|
|
return self.response.status
|
|
|
|
|
|
class Base(object):
|
|
def __str__(self):
|
|
return self.name
|
|
|
|
def header_fields(self, required_fields, optional_fields=None):
|
|
if optional_fields is None:
|
|
optional_fields = ()
|
|
|
|
headers = dict(self.conn.response.getheaders())
|
|
ret = {}
|
|
|
|
for field in required_fields:
|
|
if field[1] not in headers:
|
|
raise ValueError("%s was not found in response header" %
|
|
(field[1]))
|
|
|
|
try:
|
|
ret[field[0]] = int(headers[field[1]])
|
|
except ValueError:
|
|
ret[field[0]] = headers[field[1]]
|
|
|
|
for field in optional_fields:
|
|
if field[1] not in headers:
|
|
continue
|
|
try:
|
|
ret[field[0]] = int(headers[field[1]])
|
|
except ValueError:
|
|
ret[field[0]] = headers[field[1]]
|
|
|
|
return ret
|
|
|
|
|
|
class Account(Base):
|
|
def __init__(self, conn, name):
|
|
self.conn = conn
|
|
self.name = str(name)
|
|
|
|
def update_metadata(self, metadata=None, cfg=None):
|
|
if metadata is None:
|
|
metadata = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
headers = dict(("X-Account-Meta-%s" % k, v)
|
|
for k, v in metadata.items())
|
|
|
|
self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg)
|
|
if not 200 <= self.conn.response.status <= 299:
|
|
raise ResponseError(self.conn.response, 'POST',
|
|
self.conn.make_path(self.path))
|
|
return True
|
|
|
|
def container(self, container_name):
|
|
return Container(self.conn, self.name, container_name)
|
|
|
|
def containers(self, hdrs=None, parms=None, cfg=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
|
|
format_type = parms.get('format', None)
|
|
if format_type not in [None, 'json', 'xml']:
|
|
raise RequestError('Invalid format: %s' % format_type)
|
|
if format_type is None and 'format' in parms:
|
|
del parms['format']
|
|
|
|
status = self.conn.make_request('GET', self.path, hdrs=hdrs,
|
|
parms=parms, cfg=cfg)
|
|
if status == 200:
|
|
if format_type == 'json':
|
|
conts = json.loads(self.conn.response.read())
|
|
for cont in conts:
|
|
cont['name'] = cont['name'].encode('utf-8')
|
|
return conts
|
|
elif format_type == 'xml':
|
|
conts = []
|
|
tree = minidom.parseString(self.conn.response.read())
|
|
for x in tree.getElementsByTagName('container'):
|
|
cont = {}
|
|
for key in ['name', 'count', 'bytes']:
|
|
cont[key] = x.getElementsByTagName(key)[0].\
|
|
childNodes[0].nodeValue
|
|
conts.append(cont)
|
|
for cont in conts:
|
|
cont['name'] = cont['name'].encode('utf-8')
|
|
return conts
|
|
else:
|
|
lines = self.conn.response.read().split('\n')
|
|
if lines and not lines[-1]:
|
|
lines = lines[:-1]
|
|
return lines
|
|
elif status == 204:
|
|
return []
|
|
|
|
raise ResponseError(self.conn.response, 'GET',
|
|
self.conn.make_path(self.path))
|
|
|
|
def delete_containers(self):
|
|
for c in listing_items(self.containers):
|
|
cont = self.container(c)
|
|
if not cont.delete_recursive():
|
|
return False
|
|
|
|
return listing_empty(self.containers)
|
|
|
|
def info(self, hdrs=None, parms=None, cfg=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
if self.conn.make_request('HEAD', self.path, hdrs=hdrs,
|
|
parms=parms, cfg=cfg) != 204:
|
|
|
|
raise ResponseError(self.conn.response, 'HEAD',
|
|
self.conn.make_path(self.path))
|
|
|
|
fields = [['object_count', 'x-account-object-count'],
|
|
['container_count', 'x-account-container-count'],
|
|
['bytes_used', 'x-account-bytes-used']]
|
|
|
|
return self.header_fields(fields)
|
|
|
|
@property
|
|
def path(self):
|
|
return []
|
|
|
|
|
|
class Container(Base):
|
|
def __init__(self, conn, account, name):
|
|
self.conn = conn
|
|
self.account = str(account)
|
|
self.name = str(name)
|
|
|
|
def create(self, hdrs=None, parms=None, cfg=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
return self.conn.make_request('PUT', self.path, hdrs=hdrs,
|
|
parms=parms, cfg=cfg) in (201, 202)
|
|
|
|
def delete(self, hdrs=None, parms=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
return self.conn.make_request('DELETE', self.path, hdrs=hdrs,
|
|
parms=parms) == 204
|
|
|
|
def delete_files(self):
|
|
for f in listing_items(self.files):
|
|
file_item = self.file(f)
|
|
if not file_item.delete():
|
|
return False
|
|
|
|
return listing_empty(self.files)
|
|
|
|
def delete_recursive(self):
|
|
return self.delete_files() and self.delete()
|
|
|
|
def file(self, file_name):
|
|
return File(self.conn, self.account, self.name, file_name)
|
|
|
|
def files(self, hdrs=None, parms=None, cfg=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
format_type = parms.get('format', None)
|
|
if format_type not in [None, 'json', 'xml']:
|
|
raise RequestError('Invalid format: %s' % format_type)
|
|
if format_type is None and 'format' in parms:
|
|
del parms['format']
|
|
|
|
status = self.conn.make_request('GET', self.path, hdrs=hdrs,
|
|
parms=parms, cfg=cfg)
|
|
if status == 200:
|
|
if format_type == 'json':
|
|
files = json.loads(self.conn.response.read())
|
|
|
|
for file_item in files:
|
|
file_item['name'] = file_item['name'].encode('utf-8')
|
|
file_item['content_type'] = file_item['content_type'].\
|
|
encode('utf-8')
|
|
return files
|
|
elif format_type == 'xml':
|
|
files = []
|
|
tree = minidom.parseString(self.conn.response.read())
|
|
for x in tree.getElementsByTagName('object'):
|
|
file_item = {}
|
|
for key in ['name', 'hash', 'bytes', 'content_type',
|
|
'last_modified']:
|
|
|
|
file_item[key] = x.getElementsByTagName(key)[0].\
|
|
childNodes[0].nodeValue
|
|
files.append(file_item)
|
|
|
|
for file_item in files:
|
|
file_item['name'] = file_item['name'].encode('utf-8')
|
|
file_item['content_type'] = file_item['content_type'].\
|
|
encode('utf-8')
|
|
return files
|
|
else:
|
|
content = self.conn.response.read()
|
|
if content:
|
|
lines = content.split('\n')
|
|
if lines and not lines[-1]:
|
|
lines = lines[:-1]
|
|
return lines
|
|
else:
|
|
return []
|
|
elif status == 204:
|
|
return []
|
|
|
|
raise ResponseError(self.conn.response, 'GET',
|
|
self.conn.make_path(self.path))
|
|
|
|
def info(self, hdrs=None, parms=None, cfg=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
self.conn.make_request('HEAD', self.path, hdrs=hdrs,
|
|
parms=parms, cfg=cfg)
|
|
|
|
if self.conn.response.status == 204:
|
|
required_fields = [['bytes_used', 'x-container-bytes-used'],
|
|
['object_count', 'x-container-object-count']]
|
|
optional_fields = [['versions', 'x-versions-location']]
|
|
|
|
return self.header_fields(required_fields, optional_fields)
|
|
|
|
raise ResponseError(self.conn.response, 'HEAD',
|
|
self.conn.make_path(self.path))
|
|
|
|
@property
|
|
def path(self):
|
|
return [self.name]
|
|
|
|
|
|
class File(Base):
|
|
def __init__(self, conn, account, container, name):
|
|
self.conn = conn
|
|
self.account = str(account)
|
|
self.container = str(container)
|
|
self.name = str(name)
|
|
|
|
self.chunked_write_in_progress = False
|
|
self.content_type = None
|
|
self.size = None
|
|
self.metadata = {}
|
|
|
|
def make_headers(self, cfg=None):
|
|
if cfg is None:
|
|
cfg = {}
|
|
headers = {}
|
|
if not cfg.get('no_content_length'):
|
|
if cfg.get('set_content_length'):
|
|
headers['Content-Length'] = cfg.get('set_content_length')
|
|
elif self.size:
|
|
headers['Content-Length'] = self.size
|
|
else:
|
|
headers['Content-Length'] = 0
|
|
|
|
if cfg.get('no_content_type'):
|
|
pass
|
|
elif self.content_type:
|
|
headers['Content-Type'] = self.content_type
|
|
else:
|
|
headers['Content-Type'] = 'application/octet-stream'
|
|
|
|
for key in self.metadata:
|
|
headers['X-Object-Meta-' + key] = self.metadata[key]
|
|
|
|
return headers
|
|
|
|
@classmethod
|
|
def compute_md5sum(cls, data):
|
|
block_size = 4096
|
|
|
|
if isinstance(data, str):
|
|
data = StringIO.StringIO(data)
|
|
|
|
checksum = hashlib.md5()
|
|
buff = data.read(block_size)
|
|
while buff:
|
|
checksum.update(buff)
|
|
buff = data.read(block_size)
|
|
data.seek(0)
|
|
return checksum.hexdigest()
|
|
|
|
def copy(self, dest_cont, dest_file, hdrs=None, parms=None, cfg=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
if 'destination' in cfg:
|
|
headers = {'Destination': cfg['destination']}
|
|
elif cfg.get('no_destination'):
|
|
headers = {}
|
|
else:
|
|
headers = {'Destination': '%s/%s' % (dest_cont, dest_file)}
|
|
headers.update(hdrs)
|
|
|
|
if 'Destination' in headers:
|
|
headers['Destination'] = urllib.quote(headers['Destination'])
|
|
|
|
return self.conn.make_request('COPY', self.path, hdrs=headers,
|
|
parms=parms) == 201
|
|
|
|
def delete(self, hdrs=None, parms=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if self.conn.make_request('DELETE', self.path, hdrs=hdrs,
|
|
parms=parms) != 204:
|
|
|
|
raise ResponseError(self.conn.response, 'DELETE',
|
|
self.conn.make_path(self.path))
|
|
|
|
return True
|
|
|
|
def info(self, hdrs=None, parms=None, cfg=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
if self.conn.make_request('HEAD', self.path, hdrs=hdrs,
|
|
parms=parms, cfg=cfg) != 200:
|
|
|
|
raise ResponseError(self.conn.response, 'HEAD',
|
|
self.conn.make_path(self.path))
|
|
|
|
fields = [['content_length', 'content-length'],
|
|
['content_type', 'content-type'],
|
|
['last_modified', 'last-modified'],
|
|
['etag', 'etag']]
|
|
|
|
header_fields = self.header_fields(fields)
|
|
header_fields['etag'] = header_fields['etag'].strip('"')
|
|
return header_fields
|
|
|
|
def initialize(self, hdrs=None, parms=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if not self.name:
|
|
return False
|
|
|
|
status = self.conn.make_request('HEAD', self.path, hdrs=hdrs,
|
|
parms=parms)
|
|
if status == 404:
|
|
return False
|
|
elif (status < 200) or (status > 299):
|
|
raise ResponseError(self.conn.response, 'HEAD',
|
|
self.conn.make_path(self.path))
|
|
|
|
for hdr in self.conn.response.getheaders():
|
|
if hdr[0].lower() == 'content-type':
|
|
self.content_type = hdr[1]
|
|
if hdr[0].lower().startswith('x-object-meta-'):
|
|
self.metadata[hdr[0][14:]] = hdr[1]
|
|
if hdr[0].lower() == 'etag':
|
|
self.etag = hdr[1].strip('"')
|
|
if hdr[0].lower() == 'content-length':
|
|
self.size = int(hdr[1])
|
|
if hdr[0].lower() == 'last-modified':
|
|
self.last_modified = hdr[1]
|
|
|
|
return True
|
|
|
|
def load_from_filename(self, filename, callback=None):
|
|
fobj = open(filename, 'rb')
|
|
self.write(fobj, callback=callback)
|
|
fobj.close()
|
|
|
|
@property
|
|
def path(self):
|
|
return [self.container, self.name]
|
|
|
|
@classmethod
|
|
def random_data(cls, size=None):
|
|
if size is None:
|
|
size = random.randint(1, 32768)
|
|
fd = open('/dev/urandom', 'r')
|
|
data = fd.read(size)
|
|
fd.close()
|
|
return data
|
|
|
|
def read(self, size=-1, offset=0, hdrs=None, buffer=None,
|
|
callback=None, cfg=None, parms=None):
|
|
if cfg is None:
|
|
cfg = {}
|
|
if parms is None:
|
|
parms = {}
|
|
|
|
if size > 0:
|
|
range_string = 'bytes=%d-%d' % (offset, (offset + size) - 1)
|
|
if hdrs:
|
|
hdrs['Range'] = range_string
|
|
else:
|
|
hdrs = {'Range': range_string}
|
|
|
|
status = self.conn.make_request('GET', self.path, hdrs=hdrs,
|
|
cfg=cfg, parms=parms)
|
|
|
|
if (status < 200) or (status > 299):
|
|
raise ResponseError(self.conn.response, 'GET',
|
|
self.conn.make_path(self.path))
|
|
|
|
for hdr in self.conn.response.getheaders():
|
|
if hdr[0].lower() == 'content-type':
|
|
self.content_type = hdr[1]
|
|
|
|
if hasattr(buffer, 'write'):
|
|
scratch = self.conn.response.read(8192)
|
|
transferred = 0
|
|
|
|
while len(scratch) > 0:
|
|
buffer.write(scratch)
|
|
transferred += len(scratch)
|
|
if callable(callback):
|
|
callback(transferred, self.size)
|
|
scratch = self.conn.response.read(8192)
|
|
return None
|
|
else:
|
|
return self.conn.response.read()
|
|
|
|
def read_md5(self):
|
|
status = self.conn.make_request('GET', self.path)
|
|
|
|
if (status < 200) or (status > 299):
|
|
raise ResponseError(self.conn.response, 'GET',
|
|
self.conn.make_path(self.path))
|
|
|
|
checksum = hashlib.md5()
|
|
|
|
scratch = self.conn.response.read(8192)
|
|
while len(scratch) > 0:
|
|
checksum.update(scratch)
|
|
scratch = self.conn.response.read(8192)
|
|
|
|
return checksum.hexdigest()
|
|
|
|
def save_to_filename(self, filename, callback=None):
|
|
try:
|
|
fobj = open(filename, 'wb')
|
|
self.read(buffer=fobj, callback=callback)
|
|
finally:
|
|
fobj.close()
|
|
|
|
def sync_metadata(self, metadata=None, cfg=None):
|
|
if metadata is None:
|
|
metadata = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
|
|
self.metadata.update(metadata)
|
|
|
|
if self.metadata:
|
|
headers = self.make_headers(cfg=cfg)
|
|
if not cfg.get('no_content_length'):
|
|
if cfg.get('set_content_length'):
|
|
headers['Content-Length'] = \
|
|
cfg.get('set_content_length')
|
|
else:
|
|
headers['Content-Length'] = 0
|
|
|
|
self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg)
|
|
|
|
if self.conn.response.status not in (201, 202):
|
|
raise ResponseError(self.conn.response, 'POST',
|
|
self.conn.make_path(self.path))
|
|
|
|
return True
|
|
|
|
def chunked_write(self, data=None, hdrs=None, parms=None, cfg=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
|
|
if data is not None and self.chunked_write_in_progress:
|
|
self.conn.put_data(data, True)
|
|
elif data is not None:
|
|
self.chunked_write_in_progress = True
|
|
|
|
headers = self.make_headers(cfg=cfg)
|
|
headers.update(hdrs)
|
|
|
|
self.conn.put_start(self.path, hdrs=headers, parms=parms,
|
|
cfg=cfg, chunked=True)
|
|
|
|
self.conn.put_data(data, True)
|
|
elif self.chunked_write_in_progress:
|
|
self.chunked_write_in_progress = False
|
|
return self.conn.put_end(True) == 201
|
|
else:
|
|
raise RuntimeError
|
|
|
|
def write(self, data='', hdrs=None, parms=None, callback=None, cfg=None,
|
|
return_resp=False):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
|
|
block_size = 2 ** 20
|
|
|
|
if isinstance(data, file):
|
|
try:
|
|
data.flush()
|
|
data.seek(0)
|
|
except IOError:
|
|
pass
|
|
self.size = int(os.fstat(data.fileno())[6])
|
|
else:
|
|
data = StringIO.StringIO(data)
|
|
self.size = data.len
|
|
|
|
headers = self.make_headers(cfg=cfg)
|
|
headers.update(hdrs)
|
|
|
|
self.conn.put_start(self.path, hdrs=headers, parms=parms, cfg=cfg)
|
|
|
|
transferred = 0
|
|
buff = data.read(block_size)
|
|
buff_len = len(buff)
|
|
try:
|
|
while buff_len > 0:
|
|
self.conn.put_data(buff)
|
|
transferred += buff_len
|
|
if callable(callback):
|
|
callback(transferred, self.size)
|
|
buff = data.read(block_size)
|
|
buff_len = len(buff)
|
|
|
|
self.conn.put_end()
|
|
except socket.timeout as err:
|
|
raise err
|
|
|
|
if (self.conn.response.status < 200) or \
|
|
(self.conn.response.status > 299):
|
|
raise ResponseError(self.conn.response, 'PUT',
|
|
self.conn.make_path(self.path))
|
|
|
|
try:
|
|
data.seek(0)
|
|
except IOError:
|
|
pass
|
|
self.md5 = self.compute_md5sum(data)
|
|
|
|
if return_resp:
|
|
return self.conn.response
|
|
|
|
return True
|
|
|
|
def write_random(self, size=None, hdrs=None, parms=None, cfg=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
|
|
data = self.random_data(size)
|
|
if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg):
|
|
raise ResponseError(self.conn.response, 'PUT',
|
|
self.conn.make_path(self.path))
|
|
self.md5 = self.compute_md5sum(StringIO.StringIO(data))
|
|
return data
|
|
|
|
def write_random_return_resp(self, size=None, hdrs=None, parms=None,
|
|
cfg=None):
|
|
if hdrs is None:
|
|
hdrs = {}
|
|
if parms is None:
|
|
parms = {}
|
|
if cfg is None:
|
|
cfg = {}
|
|
|
|
data = self.random_data(size)
|
|
resp = self.write(data, hdrs=hdrs, parms=parms, cfg=cfg,
|
|
return_resp=True)
|
|
if not resp:
|
|
raise ResponseError(self.conn.response)
|
|
self.md5 = self.compute_md5sum(StringIO.StringIO(data))
|
|
return resp
|