openstack
/
swift
Code Issues Proposed changes

Merge master to feature/ec 

Change-Id: I3e1d725fa85bf68c3d5fb0a062c9a053064a9e66
Implements: blueprint swift-ec
This commit is contained in:
paul luse 2014-10-07 08:24:08 -07:00
parent bcaa00f25f 7528f2b221
commit 06800cbe44
48 changed files with 773 additions and 367 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.mailmap
View File

@ -67,3 +67,4 @@ Mauro Stettler <mauro.stettler@gmail.com> <mauro.stettler@gmail.com>
Pawel Palucki <pawel.palucki@gmail.com> <pawel.palucki@gmail.com>
Guang Yee <guang.yee@hp.com> <guang.yee@hp.com>
Jing Liuqing <jing.liuqing@99cloud.net> <jing.liuqing@99cloud.net>
Lorcan Browne <lorcan.browne@hp.com> <lorcan.browne@hp.com>

16
AUTHORS
View File

@ -21,6 +21,7 @@ Joe Arnold (joe@swiftstack.com)
Ionuț Arțăriși (iartarisi@suse.cz)
Christian Berendt (berendt@b1-systems.de)
Luis de Bethencourt (luis@debethencourt.com)
Keshava Bharadwaj (kb.sankethi@gmail.com)
Yummy Bian (yummy.bian@gmail.com)
Darrell Bishop (darrell@swiftstack.com)
James E. Blair (jeblair@openstack.org)
@ -28,19 +29,24 @@ Fabien Boucher (fabien.boucher@enovance.com)
Chmouel Boudjnah (chmouel@enovance.com)
Clark Boylan (clark.boylan@gmail.com)
Pádraig Brady (pbrady@redhat.com)
Lorcan Browne (lorcan.browne@hp.com)
Russell Bryant (rbryant@redhat.com)
Jay S. Bryant (jsbryant@us.ibm.com)
Brian D. Burns (iosctr@gmail.com)
Devin Carlen (devin.carlen@gmail.com)
Thierry Carrez (thierry@openstack.org)
Mahati Chamarthy (mahati.chamarthy@gmail.com)
Zap Chang (zapchang@gmail.com)
François Charlier (francois.charlier@enovance.com)
Ray Chen (oldsharp@163.com)
Brian Cline (bcline@softlayer.com)
Alistair Coles (alistair.coles@hp.com)
Brian Curtin (brian.curtin@rackspace.com)
Thiago da Silva (thiago@redhat.com)
Julien Danjou (julien@danjou.info)
Ksenia Demina (kdemina@mirantis.com)
Dan Dillinger (dan.dillinger@sonian.net)
Gerry Drudy (gerry.drudy@hp.com)
Morgan Fainberg (morgan.fainberg@gmail.com)
ZhiQiang Fan (aji.zqfan@gmail.com)
Flaper Fesp (flaper87@gmail.com)
@ -55,6 +61,7 @@ David Goetz (david.goetz@rackspace.com)
Jonathan Gonzalez V (jonathan.abdiel@gmail.com)
Joe Gordon (jogo@cloudscaling.com)
David Hadas (davidh@il.ibm.com)
Andrew Hale (andy@wwwdata.eu)
Soren Hansen (soren@linux2go.dk)
Richard (Rick) Hawkins (richard.hawkins@rackspace.com)
Doug Hellmann (doug.hellmann@dreamhost.com)
@ -67,6 +74,7 @@ Kun Huang (gareth@unitedstack.com)
Matthieu Huin (mhu@enovance.com)
Hodong Hwang (hodong.hwang@kt.com)
Motonobu Ichimura (motonobu@gmail.com)
Andreas Jaeger (aj@suse.de)
Shri Javadekar (shrinand@maginatics.com)
Iryoung Jeong (iryoung@gmail.com)
Paul Jimenez (pj@place.org)
@ -80,6 +88,7 @@ Morita Kazutaka (morita.kazutaka@gmail.com)
Josh Kearney (josh@jk0.org)
Ilya Kharin (ikharin@mirantis.com)
Dae S. Kim (dae@velatum.com)
Nathan Kinder (nkinder@redhat.com)
Eugene Kirpichov (ekirpichov@gmail.com)
Leah Klearman (lklrmn@gmail.com)
Steve Kowalik (steven@wedontsleep.org)
@ -88,6 +97,7 @@ Sushil Kumar (sushil.kumar2@globallogic.com)
Madhuri Kumari (madhuri.rai07@gmail.com)
Steven Lang (Steven.Lang@hgst.com)
Gonéri Le Bouder (goneri.lebouder@enovance.com)
John Leach (john@johnleach.co.uk)
Ed Leafe (ed.leafe@rackspace.com)
Thomas Leaman (thomas.leaman@hp.com)
Eohyung Lee (liquid@kt.com)
@ -104,6 +114,7 @@ Dragos Manolescu (dragosm@hp.com)
Steve Martinelli (stevemar@ca.ibm.com)
Juan J. Martinez (juan@memset.com)
Marcelo Martins (btorch@gmail.com)
Dolph Mathews (dolph.mathews@gmail.com)
Donagh McCabe (donagh.mccabe@hp.com)
Andy McCrae (andy.mccrae@gmail.com)
Paul McMillan (paul.mcmillan@nebula.com)
@ -117,6 +128,7 @@ Maru Newby (mnewby@internap.com)
Newptone (xingchao@unitedstack.com)
Colin Nicholson (colin.nicholson@iomart.com)
Zhenguo Niu (zhenguo@unitedstack.com)
Timothy Okwii (tokwii@cisco.com)
Matthew Oliver (matt@oliver.net.au)
Eamonn O'Toole (eamonn.otoole@hp.com)
James Page (james.page@ubuntu.com)
@ -129,17 +141,19 @@ Dieter Plaetinck (dieter@vimeo.com)
Peter Portante (peter.portante@redhat.com)
Dan Prince (dprince@redhat.com)
Felipe Reyes (freyes@tty.cl)
Matt Riedemann (mriedem@us.ibm.com)
Li Riqiang (lrqrun@gmail.com)
Rafael Rivero (rafael@cloudscaling.com)
Victor Rodionov (victor.rodionov@nexenta.com)
Aaron Rosen (arosen@nicira.com)
Brent Roskos (broskos@internap.com)
Cristian A Sanchez (cristian.a.sanchez@intel.com)
saranjan (saranjan@cisco.com)
Christian Schwede (info@cschwede.de)
Mark Seger (Mark.Seger@hp.com)
Andrew Clay Shafer (acs@parvuscaptus.com)
Chuck Short (chuck.short@canonical.com)
Michael Shuler (mshuler@gmail.com)
Thiago da Silva (thiago@redhat.com)
David Moreau Simard (dmsimard@iweb.com)
Scott Simpson (sasimpson@gmail.com)
Liu Siqi (meizu647@gmail.com)

54
CHANGELOG
View File

@ -1,3 +1,57 @@
swift (2.2.0)
* Added support for Keystone v3 auth.
Keystone v3 introduced the concept of "domains" and user names
are no longer unique across domains. Swift's Keystone integration
now requires that ACLs be set on IDs, which are unique across
domains, and further restricts setting new ACLs to only use IDs.
Please see http://swift.openstack.org/overview_auth.html for
more information on configuring Swift and Keystone together.
* Swift now supports server-side account-to-account copy. Server-
side copy in Swift requires the X-Copy-From header (on a PUT)
or the Destination header (on a COPY). To initiate an account-to-
account copy, the existing header value remains the same, but the
X-Copy-From-Account header (on a PUT) or the Destination-Account
(on a COPY) are used to indicate the proper account.
* Limit partition movement when adding a new placement tier.
When adding a new placement tier (server, zone, or region), Swift
previously attempted to move all placement partitions, regardless
of the space available on the new tier, to ensure the best possible
durability. Unfortunately, this could result in too many partitions
being moved all at once to a new tier. Swift's ring-builder now
ensures that only the correct number of placement partitions are
rebalanced, and thus makes adding capacity to the cluster more
efficient.
* Per storage policy container counts are now reported in an
account response headers.
* Swift will now reject, with a 4xx series response, GET requests
with more than 50 ranges, more than 3 overlapping ranges, or more
than 8 non-increasing ranges.
* The bind_port config setting is now required to be explicitly set.
* The object server can now use splice() for a zero-copy GET
response. This feature is enabled with the "splice" config variable
in the object server config and defaults to off. Also, this feature
only works on recent Linux kernels (AF_ALG sockets must be
supported). A zero-copy GET response can significantly reduce CPU
requirements for object servers.
* Added "--no-overlap" option to swift-dispersion populate so that
multiple runs of the tool can add coverage without overlapping
existing monitored partitions.
* swift-recon now supports filtering by region.
* Various other minor bug fixes and improvements.
swift (2.1.0)
* swift-ring-builder placement was improved to allow gradual addition

4
swift/account/server.py
View File

@ -156,7 +156,7 @@ class AccountController(object):
for key, value in req.headers.iteritems()
if is_sys_or_user_meta('account', key))
if metadata:
broker.update_metadata(metadata)
broker.update_metadata(metadata, validate_metadata=True)
if created:
return HTTPCreated(request=req)
else:
@ -249,7 +249,7 @@ class AccountController(object):
for key, value in req.headers.iteritems()
if is_sys_or_user_meta('account', key))
if metadata:
broker.update_metadata(metadata)
broker.update_metadata(metadata, validate_metadata=True)
return HTTPNoContent(request=req)
def __call__(self, env, start_response):

14
swift/cli/ringbuilder.py
View File

@ -830,10 +830,18 @@ def main(arguments=None):
builder_file, ring_file = parse_builder_ring_filename_args(argv)
if exists(builder_file):
try:
builder = RingBuilder.load(builder_file)
elif len(argv) < 3 or argv[2] not in('create', 'write_builder'):
print 'Ring Builder file does not exist: %s' % argv[1]
except exceptions.UnPicklingError as e:
print e
exit(EXIT_ERROR)
except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:
if len(argv) < 3 or argv[2] not in('create', 'write_builder'):
print e
exit(EXIT_ERROR)
except Exception as e:
print 'Problem occurred while reading builder file: %s. %s' % (
argv[1], e.message)
exit(EXIT_ERROR)
backup_dir = pathjoin(dirname(argv[1]), 'backups')

5
swift/common/constraints.py
View File

@ -101,7 +101,10 @@ FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json',
def check_metadata(req, target_type):
"""
Check metadata sent in the request headers.
Check metadata sent in the request headers. This should only check
that the metadata in the request given is valid. Checks against
account/container overall metadata should be forwarded on to its
respective server to be checked.
:param req: request object
:param target_type: str: one of: object, container, or account: indicates

34
swift/common/db.py
View File

@ -30,9 +30,11 @@ from tempfile import mkstemp
from eventlet import sleep, Timeout
import sqlite3
from swift.common.constraints import MAX_META_COUNT, MAX_META_OVERALL_SIZE
from swift.common.utils import json, Timestamp, renamer, \
mkdirs, lock_parent_directory, fallocate
from swift.common.exceptions import LockTimeout
from swift.common.swob import HTTPBadRequest
#: Whether calls will be made to preallocate disk space for database files.
@ -719,7 +721,35 @@ class DatabaseBroker(object):
metadata = {}
return metadata
def update_metadata(self, metadata_updates):
@staticmethod
def validate_metadata(metadata):
"""
Validates that metadata_falls within acceptable limits.
:param metadata: to be validated
:raises: HTTPBadRequest if MAX_META_COUNT or MAX_META_OVERALL_SIZE
is exceeded
"""
meta_count = 0
meta_size = 0
for key, (value, timestamp) in metadata.iteritems():
key = key.lower()
if value != '' and (key.startswith('x-account-meta') or
key.startswith('x-container-meta')):
prefix = 'x-account-meta-'
if key.startswith('x-container-meta-'):
prefix = 'x-container-meta-'
key = key[len(prefix):]
meta_count = meta_count + 1
meta_size = meta_size + len(key) + len(value)
if meta_count > MAX_META_COUNT:
raise HTTPBadRequest('Too many metadata items; max %d'
% MAX_META_COUNT)
if meta_size > MAX_META_OVERALL_SIZE:
raise HTTPBadRequest('Total metadata too large; max %d'
% MAX_META_OVERALL_SIZE)
def update_metadata(self, metadata_updates, validate_metadata=False):
"""
Updates the metadata dict for the database. The metadata dict values
are tuples of (value, timestamp) where the timestamp indicates when
@ -752,6 +782,8 @@ class DatabaseBroker(object):
value, timestamp = value_timestamp
if key not in md or timestamp > md[key][1]:
md[key] = value_timestamp
if validate_metadata:
DatabaseBroker.validate_metadata(md)
conn.execute('UPDATE %s_stat SET metadata = ?' % self.db_type,
(json.dumps(md),))
conn.commit()

24
swift/common/direct_client.py
View File

@ -108,7 +108,7 @@ def direct_get_account(node, part, account, marker=None, limit=None,
:param marker: marker query
:param limit: query limit
:param prefix: prefix query
:param delimeter: delimeter for the query
:param delimiter: delimiter for the query
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: a tuple of (response headers, a list of containers) The response
@ -116,11 +116,11 @@ def direct_get_account(node, part, account, marker=None, limit=None,
"""
path = '/' + account
return _get_direct_account_container(path, "Account", node, part,
account, marker=None,
limit=None, prefix=None,
delimiter=None,
conn_timeout=5,
response_timeout=15)
account, marker=marker,
limit=limit, prefix=prefix,
delimiter=delimiter,
conn_timeout=conn_timeout,
response_timeout=response_timeout)
def direct_delete_account(node, part, account, conn_timeout=5,
@ -183,7 +183,7 @@ def direct_get_container(node, part, account, container, marker=None,
:param marker: marker query
:param limit: query limit
:param prefix: prefix query
:param delimeter: delimeter for the query
:param delimiter: delimiter for the query
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: a tuple of (response headers, a list of objects) The response
@ -191,11 +191,11 @@ def direct_get_container(node, part, account, container, marker=None,
"""
path = '/%s/%s' % (account, container)
return _get_direct_account_container(path, "Container", node,
part, account, marker=None,
limit=None, prefix=None,
delimiter=None,
conn_timeout=5,
response_timeout=15)
part, account, marker=marker,
limit=limit, prefix=prefix,
delimiter=delimiter,
conn_timeout=conn_timeout,
response_timeout=response_timeout)
def direct_delete_container(node, part, account, container, conn_timeout=5,

12
swift/common/exceptions.py
View File

@ -123,6 +123,18 @@ class DuplicateDeviceError(RingBuilderError):
pass
class UnPicklingError(SwiftException):
pass
class FileNotFoundError(SwiftException):
pass
class PermissionError(SwiftException):
pass
class ListingIterError(SwiftException):
pass

4
swift/common/manager.py
View File

@ -625,7 +625,7 @@ class Server(object):
"""
conf_files = self.conf_files(**kwargs)
if not conf_files:
return []
return {}
pids = self.get_running_pids(**kwargs)
@ -645,7 +645,7 @@ class Server(object):
if already_started:
print _("%s already started...") % self.server
return []
return {}
if self.server not in START_ONCE_SERVERS:
kwargs['once'] = False

16
swift/common/middleware/ratelimit.py
View File

@ -18,8 +18,7 @@ from swift import gettext_ as _
import eventlet
from swift.common.utils import cache_from_env, get_logger, register_swift_info
from swift.proxy.controllers.base import get_container_memcache_key, \
get_account_info
from swift.proxy.controllers.base import get_account_info, get_container_info
from swift.common.memcached import MemcacheConnectionError
from swift.common.swob import Request, Response
@ -118,11 +117,10 @@ class RateLimitMiddleware(object):
self.container_listing_ratelimits = interpret_conf_limits(
conf, 'container_listing_ratelimit_')
def get_container_size(self, account_name, container_name):
def get_container_size(self, env):
rv = 0
memcache_key = get_container_memcache_key(account_name,
container_name)
container_info = self.memcache_client.get(memcache_key)
container_info = get_container_info(
env, self.app, swift_source='RL')
if isinstance(container_info, dict):
rv = container_info.get(
'object_count', container_info.get('container_size', 0))
@ -149,8 +147,7 @@ class RateLimitMiddleware(object):
if account_name and container_name and obj_name and \
req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
container_size = self.get_container_size(
account_name, container_name)
container_size = self.get_container_size(req.environ)
container_rate = get_maxrate(
self.container_ratelimits, container_size)
if container_rate:
@ -160,8 +157,7 @@ class RateLimitMiddleware(object):
if account_name and container_name and not obj_name and \
req.method == 'GET':
container_size = self.get_container_size(
account_name, container_name)
container_size = self.get_container_size(req.environ)
container_rate = get_maxrate(
self.container_listing_ratelimits, container_size)
if container_rate:

96
swift/common/middleware/x_profile/html_viewer.py
View File

@ -313,55 +313,55 @@ class HTMLViewer(object):
return empty_description, headers
try:
stats = Stats2(*log_files)
if not fulldirs:
stats.strip_dirs()
stats.sort_stats(sort)
nfl_filter_esc =\
nfl_filter.replace('(', '\(').replace(')', '\)')
amount = [nfl_filter_esc, limit] if nfl_filter_esc else [limit]
profile_html = self.generate_stats_html(stats, self.app_path,
profile_id, *amount)
description = "Profiling information is generated by using\
'%s' profiler." % self.profile_module
sort_repl = '<option value="%s">' % sort
sort_selected = '<option value="%s" selected>' % sort
sort = sort_tmpl.replace(sort_repl, sort_selected)
plist = ''.join(['<option value="%s">%s</option>' % (p, p)
for p in self.profile_log.get_all_pids()])
profile_element = string.Template(profile_tmpl).substitute(
{'profile_list': plist})
profile_repl = '<option value="%s">' % profile_id
profile_selected = '<option value="%s" selected>' % profile_id
profile_element = profile_element.replace(profile_repl,
profile_selected)
limit_repl = '<option value="%s">' % limit
limit_selected = '<option value="%s" selected>' % limit
limit = limit_tmpl.replace(limit_repl, limit_selected)
fulldirs_checked = 'checked' if fulldirs else ''
fulldirs_element = string.Template(fulldirs_tmpl).substitute(
{'fulldir_checked': fulldirs_checked})
nfl_filter_element = string.Template(nfl_filter_tmpl).\
substitute({'nfl_filter': nfl_filter})
form_elements = string.Template(formelements_tmpl).substitute(
{'description': description,
'action': url,
'profile': profile_element,
'sort': sort,
'limit': limit,
'fulldirs': fulldirs_element,
'nfl_filter': nfl_filter_element,
}
)
content = string.Template(index_tmpl).substitute(
{'formelements': form_elements,
'action': url,
'description': description,
'profilehtml': profile_html,
})
return content, headers
except:
except (IOError, ValueError):
raise DataLoadFailure(_('Can not load profile data from %s.')
% log_files)
if not fulldirs:
stats.strip_dirs()
stats.sort_stats(sort)
nfl_filter_esc =\
nfl_filter.replace('(', '\(').replace(')', '\)')
amount = [nfl_filter_esc, limit] if nfl_filter_esc else [limit]
profile_html = self.generate_stats_html(stats, self.app_path,
profile_id, *amount)
description = "Profiling information is generated by using\
'%s' profiler." % self.profile_module
sort_repl = '<option value="%s">' % sort
sort_selected = '<option value="%s" selected>' % sort
sort = sort_tmpl.replace(sort_repl, sort_selected)
plist = ''.join(['<option value="%s">%s</option>' % (p, p)
for p in self.profile_log.get_all_pids()])
profile_element = string.Template(profile_tmpl).substitute(
{'profile_list': plist})
profile_repl = '<option value="%s">' % profile_id
profile_selected = '<option value="%s" selected>' % profile_id
profile_element = profile_element.replace(profile_repl,
profile_selected)
limit_repl = '<option value="%s">' % limit
limit_selected = '<option value="%s" selected>' % limit
limit = limit_tmpl.replace(limit_repl, limit_selected)
fulldirs_checked = 'checked' if fulldirs else ''
fulldirs_element = string.Template(fulldirs_tmpl).substitute(
{'fulldir_checked': fulldirs_checked})
nfl_filter_element = string.Template(nfl_filter_tmpl).\
substitute({'nfl_filter': nfl_filter})
form_elements = string.Template(formelements_tmpl).substitute(
{'description': description,
'action': url,
'profile': profile_element,
'sort': sort,
'limit': limit,
'fulldirs': fulldirs_element,
'nfl_filter': nfl_filter_element,
}
)
content = string.Template(index_tmpl).substitute(
{'formelements': form_elements,
'action': url,
'description': description,
'profilehtml': profile_html,
})
return content, headers
def download(self, log_files, sort='time', limit=-1, nfl_filter='',
output_format='default'):
@ -438,7 +438,7 @@ class HTMLViewer(object):
file_path = nfls[0]
try:
lineno = int(nfls[1])
except:
except (TypeError, ValueError, IndexError):
lineno = 0
# for security reason, this need to be fixed.
if not file_path.endswith('.py'):

19
swift/common/middleware/xprofile.py
View File

@ -242,18 +242,13 @@ class ProfileMiddleware(object):
start_response('500 Internal Server Error', [])
return _('Error on render profiling results: %s') % ex
else:
try:
_locals = locals()
code = self.unwind and PROFILE_EXEC_EAGER or\
PROFILE_EXEC_LAZY
self.profiler.runctx(code, globals(), _locals)
app_iter = _locals['app_iter_']
self.dump_checkpoint()
return app_iter
except:
self.logger.exception(_('Error profiling code'))
finally:
pass
_locals = locals()
code = self.unwind and PROFILE_EXEC_EAGER or\
PROFILE_EXEC_LAZY
self.profiler.runctx(code, globals(), _locals)
app_iter = _locals['app_iter_']
self.dump_checkpoint()
return app_iter
def renew_profile(self):
self.profiler = get_profiler(self.profile_module)

54
swift/common/ring/builder.py
View File

@ -15,6 +15,7 @@
import bisect
import copy
import errno
import itertools
import math
import random
@ -623,6 +624,7 @@ class RingBuilder(object):
"""
self._last_part_moves = array('B', (0 for _junk in xrange(self.parts)))
self._last_part_moves_epoch = int(time())
self._set_parts_wanted()
self._reassign_parts(self._adjust_replica2part2dev_size()[0])
@ -643,6 +645,26 @@ class RingBuilder(object):
self._last_part_moves[part] = 0xff
self._last_part_moves_epoch = int(time())
def _get_available_parts(self):
"""
Returns a tuple (wanted_parts_total, dict of (tier: available parts in
other tiers) for all tiers in the ring.
Devices that have too much partitions (negative parts_wanted) are
ignored, otherwise the sum of all parts_wanted is 0 +/- rounding
errors.
"""
wanted_parts_total = 0
wanted_parts_for_tier = {}
for dev in self._iter_devs():
wanted_parts_total += max(0, dev['parts_wanted'])
for tier in tiers_for_dev(dev):
if tier not in wanted_parts_for_tier:
wanted_parts_for_tier[tier] = 0
wanted_parts_for_tier[tier] += max(0, dev['parts_wanted'])
return (wanted_parts_total, wanted_parts_for_tier)
def _gather_reassign_parts(self):
"""
Returns a list of (partition, replicas) pairs to be reassigned by
@ -671,6 +693,9 @@ class RingBuilder(object):
# currently sufficient spread out across the cluster.
spread_out_parts = defaultdict(list)
max_allowed_replicas = self._build_max_replicas_by_tier()
wanted_parts_total, wanted_parts_for_tier = \
self._get_available_parts()
moved_parts = 0
for part in xrange(self.parts):
# Only move one replica at a time if possible.
if part in removed_dev_parts:
@ -701,14 +726,20 @@ class RingBuilder(object):
rep_at_tier = 0
if tier in replicas_at_tier:
rep_at_tier = replicas_at_tier[tier]
# Only allowing parts to be gathered if
# there are wanted parts on other tiers
available_parts_for_tier = wanted_parts_total - \
wanted_parts_for_tier[tier] - moved_parts
if (rep_at_tier > max_allowed_replicas[tier] and
self._last_part_moves[part] >=
self.min_part_hours):
self.min_part_hours and
available_parts_for_tier > 0):
self._last_part_moves[part] = 0
spread_out_parts[part].append(replica)
dev['parts_wanted'] += 1
dev['parts'] -= 1
removed_replica = True
moved_parts += 1
break
if removed_replica:
if dev['id'] not in tfd:
@ -1055,7 +1086,26 @@ class RingBuilder(object):
:param builder_file: path to builder file to load
:return: RingBuilder instance
"""
builder = pickle.load(open(builder_file, 'rb'))
try:
fp = open(builder_file, 'rb')
except IOError as e:
if e.errno == errno.ENOENT:
raise exceptions.FileNotFoundError(
'Ring Builder file does not exist: %s' % builder_file)
elif e.errno in [errno.EPERM, errno.EACCES]:
raise exceptions.PermissionError(
'Ring Builder file cannot be accessed: %s' % builder_file)
else:
raise
else:
with fp:
try:
builder = pickle.load(fp)
except Exception:
# raise error during unpickling as UnPicklingError
raise exceptions.UnPicklingError(
'Ring Builder file is invalid: %s' % builder_file)
if not hasattr(builder, 'devs'):
builder_dict = builder
builder = RingBuilder(1, 1, 1)

8
swift/common/wsgi.py
View File

@ -380,6 +380,10 @@ def run_server(conf, logger, sock, global_conf=None):
eventlet.patcher.monkey_patch(all=False, socket=True)
eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
eventlet.debug.hub_exceptions(eventlet_debug)
wsgi_logger = NullLogger()
if eventlet_debug:
# let eventlet.wsgi.server log to stderr
wsgi_logger = None
# utils.LogAdapter stashes name in server; fallback on unadapted loggers
if not global_conf:
if hasattr(logger, 'server'):
@ -395,10 +399,10 @@ def run_server(conf, logger, sock, global_conf=None):
# necessary for the AWS SDK to work with swift3 middleware.
argspec = inspect.getargspec(wsgi.server)
if 'capitalize_response_headers' in argspec.args:
wsgi.server(sock, app, NullLogger(), custom_pool=pool,
wsgi.server(sock, app, wsgi_logger, custom_pool=pool,
capitalize_response_headers=False)
else:
wsgi.server(sock, app, NullLogger(), custom_pool=pool)
wsgi.server(sock, app, wsgi_logger, custom_pool=pool)
except socket.error as err:
if err[0] != errno.EINVAL:
raise

5
swift/container/reconciler.py
View File

@ -365,6 +365,7 @@ class ContainerReconciler(Daemon):
object queue entry.
:param container: the misplaced objects container
:param obj: the name of the misplaced object
:param q_ts: the timestamp of the misplaced object
:param q_record: the timestamp of the queue entry
@ -387,7 +388,7 @@ class ContainerReconciler(Daemon):
:param account: the account name
:param container: the container name
:param account: the object name
:param obj: the object name
:param timestamp: the timestamp of the object to delete
:param policy_index: the policy index to direct the request
:param path: the path to be used for logging
@ -732,7 +733,7 @@ class ContainerReconciler(Daemon):
"""
try:
self.reconcile()
except:
except: # noqa
self.logger.exception('Unhandled Exception trying to reconcile')
self.log_stats(force=True)

4
swift/container/server.py
View File

@ -374,7 +374,7 @@ class ContainerController(object):
metadata['X-Container-Sync-To'][0] != \
broker.metadata['X-Container-Sync-To'][0]:
broker.set_x_container_sync_points(-1, -1)
broker.update_metadata(metadata)
broker.update_metadata(metadata, validate_metadata=True)
resp = self.account_update(req, account, container, broker)
if resp:
return resp
@ -551,7 +551,7 @@ class ContainerController(object):
metadata['X-Container-Sync-To'][0] != \
broker.metadata['X-Container-Sync-To'][0]:
broker.set_x_container_sync_points(-1, -1)
broker.update_metadata(metadata)
broker.update_metadata(metadata, validate_metadata=True)
return HTTPNoContent(request=req)
def __call__(self, env, start_response):

21
swift/locale/en_GB/LC_MESSAGES/swift-log-critical.po
View File

@ -1,21 +0,0 @@
# Translations template for heat.
# Copyright (C) 2014 ORGANIZATION
# This file is distributed under the same license as the heat project.
#
# Translators:
# Andi Chandler <andi@gowling.com>, 2014
msgid ""
msgstr ""
"Project-Id-Version: Swift\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
"PO-Revision-Date: 2014-07-25 15:03+0000\n"
"Last-Translator: Andi Chandler <andi@gowling.com>\n"
"Language-Team: English (United Kingdom) (http://www.transifex.com/projects/p/"
"swift/language/en_GB/)\n"
"Language: en_GB\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 1.3\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"

21
swift/locale/en_GB/LC_MESSAGES/swift-log-error.po
View File

@ -1,21 +0,0 @@
# Translations template for heat.
# Copyright (C) 2014 ORGANIZATION
# This file is distributed under the same license as the heat project.
#
# Translators:
# Andi Chandler <andi@gowling.com>, 2014
msgid ""
msgstr ""
"Project-Id-Version: Swift\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
"PO-Revision-Date: 2014-07-25 23:08+0000\n"
"Last-Translator: Andi Chandler <andi@gowling.com>\n"
"Language-Team: English (United Kingdom) (http://www.transifex.com/projects/p/"
"swift/language/en_GB/)\n"
"Language: en_GB\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 1.3\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"

21
swift/locale/en_GB/LC_MESSAGES/swift-log-info.po
View File

@ -1,21 +0,0 @@
# Translations template for heat.
# Copyright (C) 2014 ORGANIZATION
# This file is distributed under the same license as the heat project.
#
# Translators:
# Andi Chandler <andi@gowling.com>, 2014
msgid ""
msgstr ""
"Project-Id-Version: Swift\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
"PO-Revision-Date: 2014-07-25 15:03+0000\n"
"Last-Translator: Andi Chandler <andi@gowling.com>\n"
"Language-Team: English (United Kingdom) (http://www.transifex.com/projects/p/"
"swift/language/en_GB/)\n"
"Language: en_GB\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 1.3\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"

21
swift/locale/en_GB/LC_MESSAGES/swift-log-warning.po
View File

@ -1,21 +0,0 @@
# Translations template for heat.
# Copyright (C) 2014 ORGANIZATION
# This file is distributed under the same license as the heat project.
#
# Translators:
# Andi Chandler <andi@gowling.com>, 2014
msgid ""
msgstr ""
"Project-Id-Version: Swift\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
"PO-Revision-Date: 2014-07-25 15:02+0000\n"
"Last-Translator: Andi Chandler <andi@gowling.com>\n"
"Language-Team: English (United Kingdom) (http://www.transifex.com/projects/p/"
"swift/language/en_GB/)\n"
"Language: en_GB\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 1.3\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"

21
swift/locale/ko_KR/LC_MESSAGES/swift-log-critical.po
View File

@ -1,21 +0,0 @@
# Translations template for heat.
# Copyright (C) 2014 ORGANIZATION
# This file is distributed under the same license as the heat project.
#
# Translators:
# Mario Cho <hephaex@gmail.com>, 2014
msgid ""
msgstr ""
"Project-Id-Version: Swift\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
"PO-Revision-Date: 2014-09-18 02:40+0000\n"
"Last-Translator: Mario Cho <hephaex@gmail.com>\n"
"Language-Team: Korean (Korea) (http://www.transifex.com/projects/p/swift/"
"language/ko_KR/)\n"
"Language: ko_KR\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 1.3\n"
"Plural-Forms: nplurals=1; plural=0;\n"

21
swift/locale/ko_KR/LC_MESSAGES/swift-log-error.po
View File

@ -1,21 +0,0 @@
# Translations template for heat.
# Copyright (C) 2014 ORGANIZATION
# This file is distributed under the same license as the heat project.
#
# Translators:
# Mario Cho <hephaex@gmail.com>, 2014
msgid ""
msgstr ""
"Project-Id-Version: Swift\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
"PO-Revision-Date: 2014-09-18 02:40+0000\n"
"Last-Translator: Mario Cho <hephaex@gmail.com>\n"
"Language-Team: Korean (Korea) (http://www.transifex.com/projects/p/swift/"
"language/ko_KR/)\n"
"Language: ko_KR\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 1.3\n"
"Plural-Forms: nplurals=1; plural=0;\n"

21
swift/locale/ko_KR/LC_MESSAGES/swift-log-info.po
View File

@ -1,21 +0,0 @@
# Translations template for heat.
# Copyright (C) 2014 ORGANIZATION
# This file is distributed under the same license as the heat project.
#
# Translators:
# Mario Cho <hephaex@gmail.com>, 2014
msgid ""
msgstr ""
"Project-Id-Version: Swift\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
"PO-Revision-Date: 2014-09-18 02:40+0000\n"
"Last-Translator: Mario Cho <hephaex@gmail.com>\n"
"Language-Team: Korean (Korea) (http://www.transifex.com/projects/p/swift/"
"language/ko_KR/)\n"
"Language: ko_KR\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 1.3\n"
"Plural-Forms: nplurals=1; plural=0;\n"

21
swift/locale/ko_KR/LC_MESSAGES/swift-log-warning.po
View File

@ -1,21 +0,0 @@
# Translations template for heat.
# Copyright (C) 2014 ORGANIZATION
# This file is distributed under the same license as the heat project.
#
# Translators:
# Mario Cho <hephaex@gmail.com>, 2014
msgid ""
msgstr ""
"Project-Id-Version: Swift\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
"PO-Revision-Date: 2014-09-18 02:40+0000\n"
"Last-Translator: Mario Cho <hephaex@gmail.com>\n"
"Language-Team: Korean (Korea) (http://www.transifex.com/projects/p/swift/"
"language/ko_KR/)\n"
"Language: ko_KR\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 1.3\n"
"Plural-Forms: nplurals=1; plural=0;\n"

96
swift/locale/swift.pot
View File

@ -6,9 +6,9 @@
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: swift 2.1.0.77.g0d0c16d\n"
"Project-Id-Version: swift 2.1.0.98.g6cd860b\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
"POT-Creation-Date: 2014-09-28 06:08+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@ -17,42 +17,54 @@ msgstr ""
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 1.3\n"
#: swift/account/auditor.py:58
#: swift/account/auditor.py:59
#, python-format
msgid ""
"Since %(time)s: Account audits: %(passed)s passed audit,%(failed)s failed"
" audit"
msgstr ""
#: swift/account/auditor.py:81
#: swift/account/auditor.py:82
msgid "Begin account audit pass."
msgstr ""
#: swift/account/auditor.py:87 swift/container/auditor.py:86
#: swift/account/auditor.py:88 swift/container/auditor.py:86
msgid "ERROR auditing"
msgstr ""
#: swift/account/auditor.py:92
#: swift/account/auditor.py:93
#, python-format
msgid "Account audit pass completed: %.02fs"
msgstr ""
#: swift/account/auditor.py:98
#: swift/account/auditor.py:99
msgid "Begin account audit \"once\" mode"
msgstr ""
#: swift/account/auditor.py:103
#: swift/account/auditor.py:104
#, python-format
msgid "Account audit \"once\" mode completed: %.02fs"
msgstr ""
#: swift/account/auditor.py:124
#: swift/account/auditor.py:123
#, python-format
msgid ""
"The total %(key)s for the container (%(total)s) does not match the sum of"
" %(key)s across policies (%(sum)s)"
msgstr ""
#: swift/account/auditor.py:149
#, python-format
msgid "Audit Failed for %s: %s"
msgstr ""
#: swift/account/auditor.py:153
#, python-format
msgid "ERROR Could not get account info %s"
msgstr ""
#: swift/account/reaper.py:132 swift/common/utils.py:1952
#: swift/obj/diskfile.py:425 swift/obj/updater.py:78 swift/obj/updater.py:121
#: swift/account/reaper.py:132 swift/common/utils.py:1970
#: swift/obj/diskfile.py:432 swift/obj/updater.py:78 swift/obj/updater.py:121
#, python-format
msgid "Skipping %s as it is not mounted"
msgstr ""
@ -142,7 +154,7 @@ msgid "Exception with objects for container %(container)s for account %(account)
msgstr ""
#: swift/account/server.py:278 swift/container/server.py:580
#: swift/obj/server.py:697
#: swift/obj/server.py:710
#, python-format
msgid "ERROR __call__ error with %(method)s %(path)s "
msgstr ""
@ -366,95 +378,95 @@ msgstr ""
msgid "Error limiting server %s"
msgstr ""
#: swift/common/utils.py:306
#: swift/common/utils.py:324
#, python-format
msgid "Unable to locate %s in libc. Leaving as a no-op."
msgstr ""
#: swift/common/utils.py:480
#: swift/common/utils.py:498
msgid "Unable to locate fallocate, posix_fallocate in libc. Leaving as a no-op."
msgstr ""
#: swift/common/utils.py:911
#: swift/common/utils.py:929
msgid "STDOUT: Connection reset by peer"
msgstr ""
#: swift/common/utils.py:913 swift/common/utils.py:916
#: swift/common/utils.py:931 swift/common/utils.py:934
#, python-format
msgid "STDOUT: %s"
msgstr ""
#: swift/common/utils.py:1150
#: swift/common/utils.py:1168
msgid "Connection refused"
msgstr ""
#: swift/common/utils.py:1152
#: swift/common/utils.py:1170
msgid "Host unreachable"
msgstr ""
#: swift/common/utils.py:1154
#: swift/common/utils.py:1172
msgid "Connection timeout"
msgstr ""
#: swift/common/utils.py:1456
#: swift/common/utils.py:1474
msgid "UNCAUGHT EXCEPTION"
msgstr ""
#: swift/common/utils.py:1511
#: swift/common/utils.py:1529
msgid "Error: missing config path argument"
msgstr ""
#: swift/common/utils.py:1516
#: swift/common/utils.py:1534
#, python-format
msgid "Error: unable to locate %s"
msgstr ""
#: swift/common/utils.py:1813
#: swift/common/utils.py:1831
#, python-format
msgid "Unable to read config from %s"
msgstr ""
#: swift/common/utils.py:1819
#: swift/common/utils.py:1837
#, python-format
msgid "Unable to find %s config section in %s"
msgstr ""
#: swift/common/utils.py:2173
#: swift/common/utils.py:2191
#, python-format
msgid "Invalid X-Container-Sync-To format %r"
msgstr ""
#: swift/common/utils.py:2178
#: swift/common/utils.py:2196
#, python-format
msgid "No realm key for %r"
msgstr ""
#: swift/common/utils.py:2182
#: swift/common/utils.py:2200
#, python-format
msgid "No cluster endpoint for %r %r"
msgstr ""
#: swift/common/utils.py:2191
#: swift/common/utils.py:2209
#, python-format
msgid ""
"Invalid scheme %r in X-Container-Sync-To, must be \"//\", \"http\", or "
"\"https\"."
msgstr ""
#: swift/common/utils.py:2195
#: swift/common/utils.py:2213
msgid "Path required in X-Container-Sync-To"
msgstr ""
#: swift/common/utils.py:2198
#: swift/common/utils.py:2216
msgid "Params, queries, and fragments not allowed in X-Container-Sync-To"
msgstr ""
#: swift/common/utils.py:2203
#: swift/common/utils.py:2221
#, python-format
msgid "Invalid host %r in X-Container-Sync-To"
msgstr ""
#: swift/common/utils.py:2395
#: swift/common/utils.py:2413
msgid "Exception dumping recon cache"
msgstr ""
@ -793,36 +805,36 @@ msgstr ""
msgid "ERROR auditing: %s"
msgstr ""
#: swift/obj/diskfile.py:275
#: swift/obj/diskfile.py:282
#, python-format
msgid "Quarantined %(hsh_path)s to %(quar_path)s because it is not a directory"
msgstr ""
#: swift/obj/diskfile.py:364
#: swift/obj/diskfile.py:371
msgid "Error hashing suffix"
msgstr ""
#: swift/obj/diskfile.py:439 swift/obj/updater.py:160
#: swift/obj/diskfile.py:446 swift/obj/updater.py:160
#, python-format
msgid "Directory %s does not map to a valid policy"
msgstr ""
#: swift/obj/diskfile.py:602
#: swift/obj/diskfile.py:642
#, python-format
msgid "Quarantined %(object_path)s to %(quar_path)s because it is not a directory"
msgstr ""
#: swift/obj/diskfile.py:784
#: swift/obj/diskfile.py:824
#, python-format
msgid "Problem cleaning up %s"
msgstr ""
#: swift/obj/diskfile.py:969
#: swift/obj/diskfile.py:1120
#, python-format
msgid "ERROR DiskFile %(data_file)s close failure: %(exc)s : %(stack)s"
msgstr ""
#: swift/obj/diskfile.py:1246
#: swift/obj/diskfile.py:1401
#, python-format
msgid ""
"Client path %(client)s does not match path stored in object metadata "
@ -971,21 +983,21 @@ msgstr ""
msgid "Object replication complete. (%.02f minutes)"
msgstr ""
#: swift/obj/server.py:188
#: swift/obj/server.py:201
#, python-format
msgid ""
"ERROR Container update failed (saving for async update later): %(status)d"
" response from %(ip)s:%(port)s/%(dev)s"
msgstr ""
#: swift/obj/server.py:195
#: swift/obj/server.py:208
#, python-format
msgid ""
"ERROR container update failed with %(ip)s:%(port)s/%(dev)s (saving for "
"async update later)"
msgstr ""
#: swift/obj/server.py:230
#: swift/obj/server.py:243
#, python-format
msgid ""
"ERROR Container update failed: different numbers of hosts and devices in "

8
swift/obj/auditor.py
View File

@ -257,8 +257,12 @@ class ObjectAuditor(Daemon):
signal.signal(signal.SIGTERM, signal.SIG_DFL)
if zero_byte_fps:
kwargs['zero_byte_fps'] = self.conf_zero_byte_fps
self.run_audit(**kwargs)
sys.exit()
try:
self.run_audit(**kwargs)
except Exception as e:
self.logger.error(_("ERROR: Unable to run auditing: %s") % e)
finally:
sys.exit()
def audit_loop(self, parent, zbo_fps, override_devices=None, **kwargs):
"""Parallel audit loop"""

2
swift/obj/diskfile.py
View File

@ -358,7 +358,7 @@ def get_hashes(partition_dir, recalculate=None, do_listdir=False,
if len(suff) == 3:
hashes.setdefault(suff, None)
modified = True
hashes.update((hash_, None) for hash_ in recalculate)
hashes.update((suffix, None) for suffix in recalculate)
for suffix, hash_ in hashes.items():
if not hash_:
suffix_dir = join(partition_dir, suffix)

3
swift/obj/replicator.py
View File

@ -83,7 +83,6 @@ class ObjectReplicator(Daemon):
self.node_timeout = float(conf.get('node_timeout', 10))
self.sync_method = getattr(self, conf.get('sync_method') or 'rsync')
self.network_chunk_size = int(conf.get('network_chunk_size', 65536))
self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536))
self.headers = {
'Content-Length': '0',
'user-agent': 'object-replicator %s' % os.getpid()}
@ -284,7 +283,7 @@ class ObjectReplicator(Daemon):
job['nodes'],
job['object_ring'].get_more_nodes(int(job['partition'])))
while attempts_left > 0:
# If this throws StopIterator it will be caught way below
# If this throws StopIteration it will be caught way below
node = next(nodes)
attempts_left -= 1
try:

25
test/functional/test_account.py
View File

@ -774,6 +774,21 @@ class TestAccount(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 400)
def test_bad_metadata2(self):
if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
# TODO: Find the test that adds these and remove them.
headers = {'x-remove-account-meta-temp-url-key': 'remove',
'x-remove-account-meta-temp-url-key-2': 'remove'}
resp = retry(post, headers)
headers = {}
for x in xrange(self.max_meta_count):
headers['X-Account-Meta-%d' % x] = 'v'
@ -787,6 +802,16 @@ class TestAccount(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 400)
def test_bad_metadata3(self):
if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
headers = {}
header_value = 'k' * self.max_meta_value_length
size = 0

20
test/functional/test_container.py
View File

@ -401,6 +401,16 @@ class TestContainer(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 400)
def test_POST_bad_metadata2(self):
if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
conn.request('POST', parsed.path + '/' + self.name, '', headers)
return check_response(conn)
headers = {}
for x in xrange(self.max_meta_count):
headers['X-Container-Meta-%d' % x] = 'v'
@ -414,6 +424,16 @@ class TestContainer(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 400)
def test_POST_bad_metadata3(self):
if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
conn.request('POST', parsed.path + '/' + self.name, '', headers)
return check_response(conn)
headers = {}
header_value = 'k' * self.max_meta_value_length
size = 0

5
test/unit/cli/test_recon.py
View File

@ -99,7 +99,7 @@ class TestRecon(unittest.TestCase):
def tearDown(self, *_args, **_kwargs):
try:
os.remove(self.tmpfile_name)
except:
except OSError:
pass
def test_gen_stats(self):
@ -208,6 +208,9 @@ class TestRecon(unittest.TestCase):
self.fail('Did not find expected substring %r '
'in output:\n%s' % (expected, output))
for ring in ('account', 'container', 'object', 'object-1'):
os.remove(os.path.join(self.swift_dir, "%s.ring.gz" % ring))
class TestReconCommands(unittest.TestCase):
def setUp(self):

62
test/unit/cli/test_ringbuilder.py
View File

@ -13,11 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import os
import tempfile
import unittest
import uuid
import swift.cli.ringbuilder
from swift.common import exceptions
from swift.common.ring import RingBuilder
@ -199,6 +202,65 @@ class TestCommands(unittest.TestCase):
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.replicas, 3.14159265359)
def test_validate(self):
self.create_sample_ring()
ring = RingBuilder.load(self.tmpfile)
ring.rebalance()
ring.save(self.tmpfile)
argv = ["", self.tmpfile, "validate"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
def test_validate_empty_file(self):
open(self.tmpfile, 'a').close
argv = ["", self.tmpfile, "validate"]
try:
swift.cli.ringbuilder.main(argv)
except SystemExit as e:
self.assertEquals(e.code, 2)
def test_validate_corrupted_file(self):
self.create_sample_ring()
ring = RingBuilder.load(self.tmpfile)
ring.rebalance()
self.assertTrue(ring.validate()) # ring is valid until now
ring.save(self.tmpfile)
argv = ["", self.tmpfile, "validate"]
# corrupt the file
with open(self.tmpfile, 'wb') as f:
f.write(os.urandom(1024))
try:
swift.cli.ringbuilder.main(argv)
except SystemExit as e:
self.assertEquals(e.code, 2)
def test_validate_non_existent_file(self):
rand_file = '%s/%s' % ('/tmp', str(uuid.uuid4()))
argv = ["", rand_file, "validate"]
try:
swift.cli.ringbuilder.main(argv)
except SystemExit as e:
self.assertEquals(e.code, 2)
def test_validate_non_accessible_file(self):
with mock.patch.object(
RingBuilder, 'load',
mock.Mock(side_effect=exceptions.PermissionError)):
argv = ["", self.tmpfile, "validate"]
try:
swift.cli.ringbuilder.main(argv)
except SystemExit as e:
self.assertEquals(e.code, 2)
def test_validate_generic_error(self):
with mock.patch.object(RingBuilder, 'load',
mock.Mock(side_effect=
IOError('Generic error occurred'))):
argv = ["", self.tmpfile, "validate"]
try:
swift.cli.ringbuilder.main(argv)
except SystemExit as e:
self.assertEquals(e.code, 2)
if __name__ == '__main__':
unittest.main()

13
test/unit/common/middleware/test_dlo.py
View File

@ -18,13 +18,16 @@ import contextlib
import hashlib
import json
import mock
import shutil
import tempfile
from textwrap import dedent
import time
import unittest
from swift.common import exceptions, swob
from swift.common.middleware import dlo
from test.unit.common.middleware.helpers import FakeSwift
from textwrap import dedent
LIMIT = 'swift.common.constraints.CONTAINER_LISTING_LIMIT'
@ -898,6 +901,12 @@ class TestDloConfiguration(unittest.TestCase):
proxy's config section if we don't have any config values.
"""
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_skip_defaults_if_configured(self):
# The presence of even one config value in our config section means we
# won't go looking for the proxy config at all.
@ -984,7 +993,7 @@ class TestDloConfiguration(unittest.TestCase):
max_get_time = 2900
""")
conf_dir = tempfile.mkdtemp()
conf_dir = self.tmpdir
conffile1 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf')
conffile1.write(proxy_conf1)

37
test/unit/common/middleware/test_ratelimit.py
View File

@ -189,7 +189,7 @@ class TestRateLimit(unittest.TestCase):
the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
the_app.memcache_client = fake_memcache
req = lambda: None
req.environ = {}
req.environ = {'swift.cache': fake_memcache, 'PATH_INFO': '/v1/a/c/o'}
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
req.method = 'DELETE'
@ -243,7 +243,7 @@ class TestRateLimit(unittest.TestCase):
the_app.memcache_client = fake_memcache
req = lambda: None
req.method = 'PUT'
req.environ = {}
req.environ = {'PATH_INFO': '/v1/a/c/o', 'swift.cache': fake_memcache}
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o')
@ -255,20 +255,23 @@ class TestRateLimit(unittest.TestCase):
conf_dict = {'account_ratelimit': current_rate}
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
ratelimit.http_connect = mock_http_connect(204)
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
with mock.patch('swift.common.middleware.ratelimit.get_container_info',
lambda *args, **kwargs: {}):
for meth, exp_time in [
('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]:
req = Request.blank('/v/a%s/c' % meth)
req.method = meth
req.environ['swift.cache'] = FakeMemcache()
make_app_call = lambda: self.test_ratelimit(req.environ,
start_response)
begin = time.time()
self._run(make_app_call, num_calls, current_rate,
check_time=bool(exp_time))
self.assertEquals(round(time.time() - begin, 1), exp_time)
self._reset_time()
with mock.patch(
'swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
for meth, exp_time in [('DELETE', 9.8), ('GET', 0),
('POST', 0), ('PUT', 9.8)]:
req = Request.blank('/v/a%s/c' % meth)
req.method = meth
req.environ['swift.cache'] = FakeMemcache()
make_app_call = lambda: self.test_ratelimit(req.environ,
start_response)
begin = time.time()
self._run(make_app_call, num_calls, current_rate,
check_time=bool(exp_time))
self.assertEquals(round(time.time() - begin, 1), exp_time)
self._reset_time()
def test_ratelimit_set_incr(self):
current_rate = 5
@ -403,7 +406,7 @@ class TestRateLimit(unittest.TestCase):
req.method = 'PUT'
req.environ['swift.cache'] = FakeMemcache()
req.environ['swift.cache'].set(
ratelimit.get_container_memcache_key('a', 'c'),
get_container_memcache_key('a', 'c'),
{'container_size': 1})
time_override = [0, 0, 0, 0, None]
@ -437,7 +440,7 @@ class TestRateLimit(unittest.TestCase):
req.method = 'GET'
req.environ['swift.cache'] = FakeMemcache()
req.environ['swift.cache'].set(
ratelimit.get_container_memcache_key('a', 'c'),
get_container_memcache_key('a', 'c'),
{'container_size': 1})
time_override = [0, 0, 0, 0, None]

12
test/unit/common/middleware/test_xprofile.py
View File

@ -195,8 +195,9 @@ class Test_profile_log(unittest.TestCase):
def setUp(self):
if xprofile is None:
raise SkipTest
self.tempdirs = [tempfile.mkdtemp(), tempfile.mkdtemp()]
self.log_filename_prefix1 = self.tempdirs[0] + '/unittest.profile'
self.dir1 = tempfile.mkdtemp()
self.log_filename_prefix1 = self.dir1 + '/unittest.profile'
self.profile_log1 = ProfileLog(self.log_filename_prefix1, False)
self.pids1 = ['123', '456', str(os.getpid())]
profiler1 = xprofile.get_profiler('eventlet.green.profile')
@ -204,7 +205,8 @@ class Test_profile_log(unittest.TestCase):
profiler1.runctx('import os;os.getcwd();', globals(), locals())
self.profile_log1.dump_profile(profiler1, pid)
self.log_filename_prefix2 = self.tempdirs[1] + '/unittest.profile'
self.dir2 = tempfile.mkdtemp()
self.log_filename_prefix2 = self.dir2 + '/unittest.profile'
self.profile_log2 = ProfileLog(self.log_filename_prefix2, True)
self.pids2 = ['321', '654', str(os.getpid())]
profiler2 = xprofile.get_profiler('eventlet.green.profile')
@ -215,8 +217,8 @@ class Test_profile_log(unittest.TestCase):
def tearDown(self):
self.profile_log1.clear('all')
self.profile_log2.clear('all')
for tempdir in self.tempdirs:
shutil.rmtree(tempdir, ignore_errors=True)
shutil.rmtree(self.dir1, ignore_errors=True)
shutil.rmtree(self.dir2, ignore_errors=True)
def test_get_all_pids(self):
self.assertEquals(self.profile_log1.get_all_pids(),

96
test/unit/common/ring/test_builder.py
View File

@ -13,12 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import errno
import mock
import operator
import os
import unittest
import cPickle as pickle
from collections import defaultdict
from math import ceil
from tempfile import mkdtemp
from shutil import rmtree
@ -718,9 +720,7 @@ class TestRingBuilder(unittest.TestCase):
population_by_region = self._get_population_by_region(rb)
self.assertEquals(population_by_region, {0: 682, 1: 86})
# Rebalancing will reassign 143 of the partitions, which is ~1/5
# of the total amount of partitions (3*256)
self.assertEqual(143, changed_parts)
self.assertEqual(87, changed_parts)
# and since there's not enough room, subsequent rebalances will not
# cause additional assignments to r1
@ -744,6 +744,35 @@ class TestRingBuilder(unittest.TestCase):
population_by_region = self._get_population_by_region(rb)
self.assertEquals(population_by_region, {0: 512, 1: 256})
def test_avoid_tier_change_new_region(self):
rb = ring.RingBuilder(8, 3, 1)
for i in range(5):
rb.add_dev({'id': i, 'region': 0, 'zone': 0, 'weight': 100,
'ip': '127.0.0.1', 'port': i, 'device': 'sda1'})
rb.rebalance(seed=2)
# Add a new device in new region to a balanced ring
rb.add_dev({'id': 5, 'region': 1, 'zone': 0, 'weight': 0,
'ip': '127.0.0.5', 'port': 10000, 'device': 'sda1'})
# Increase the weight of region 1 slowly
moved_partitions = []
for weight in range(0, 101, 10):
rb.set_dev_weight(5, weight)
rb.pretend_min_part_hours_passed()
changed_parts, _balance = rb.rebalance(seed=2)
moved_partitions.append(changed_parts)
# Ensure that the second region has enough partitions
# Otherwise there will be replicas at risk
min_parts_for_r1 = ceil(weight / (500.0 + weight) * 768)
parts_for_r1 = self._get_population_by_region(rb).get(1, 0)
self.assertEqual(min_parts_for_r1, parts_for_r1)
# Number of partitions moved on each rebalance
# 10/510 * 768 ~ 15.06 -> move at least 15 partitions in first step
ref = [0, 17, 16, 16, 14, 15, 13, 13, 12, 12, 14]
self.assertEqual(ref, moved_partitions)
def test_set_replicas_increase(self):
rb = ring.RingBuilder(8, 2, 0)
rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
@ -801,6 +830,14 @@ class TestRingBuilder(unittest.TestCase):
self.assertEqual([len(p2d) for p2d in rb._replica2part2dev],
[256, 256, 128])
def test_create_add_dev_add_replica_rebalance(self):
rb = ring.RingBuilder(8, 3, 1)
rb.add_dev({'id': 0, 'region': 0, 'region': 0, 'zone': 0, 'weight': 3,
'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'})
rb.set_replicas(4)
rb.rebalance() # this would crash since parts_wanted was not set
rb.validate()
def test_add_replicas_then_rebalance_respects_weight(self):
rb = ring.RingBuilder(8, 3, 1)
rb.add_dev({'id': 0, 'region': 0, 'region': 0, 'zone': 0, 'weight': 3,
@ -877,17 +914,25 @@ class TestRingBuilder(unittest.TestCase):
rb.rebalance()
real_pickle = pickle.load
fake_open = mock.mock_open()
io_error_not_found = IOError()
io_error_not_found.errno = errno.ENOENT
io_error_no_perm = IOError()
io_error_no_perm.errno = errno.EPERM
io_error_generic = IOError()
io_error_generic.errno = errno.EOPNOTSUPP
try:
#test a legit builder
fake_pickle = mock.Mock(return_value=rb)
fake_open = mock.Mock(return_value=None)
pickle.load = fake_pickle
builder = ring.RingBuilder.load('fake.builder', open=fake_open)
self.assertEquals(fake_pickle.call_count, 1)
fake_open.assert_has_calls([mock.call('fake.builder', 'rb')])
self.assertEquals(builder, rb)
fake_pickle.reset_mock()
fake_open.reset_mock()
#test old style builder
fake_pickle.return_value = rb.to_dict()
@ -896,7 +941,6 @@ class TestRingBuilder(unittest.TestCase):
fake_open.assert_has_calls([mock.call('fake.builder', 'rb')])
self.assertEquals(builder.devs, rb.devs)
fake_pickle.reset_mock()
fake_open.reset_mock()
#test old devs but no meta
no_meta_builder = rb
@ -907,10 +951,48 @@ class TestRingBuilder(unittest.TestCase):
builder = ring.RingBuilder.load('fake.builder', open=fake_open)
fake_open.assert_has_calls([mock.call('fake.builder', 'rb')])
self.assertEquals(builder.devs, rb.devs)
fake_pickle.reset_mock()
#test an empty builder
fake_pickle.side_effect = EOFError
pickle.load = fake_pickle
self.assertRaises(exceptions.UnPicklingError,
ring.RingBuilder.load, 'fake.builder',
open=fake_open)
#test a corrupted builder
fake_pickle.side_effect = pickle.UnpicklingError
pickle.load = fake_pickle
self.assertRaises(exceptions.UnPicklingError,
ring.RingBuilder.load, 'fake.builder',
open=fake_open)
#test some error
fake_pickle.side_effect = AttributeError
pickle.load = fake_pickle
self.assertRaises(exceptions.UnPicklingError,
ring.RingBuilder.load, 'fake.builder',
open=fake_open)
finally:
pickle.load = real_pickle
#test non existent builder file
fake_open.side_effect = io_error_not_found
self.assertRaises(exceptions.FileNotFoundError,
ring.RingBuilder.load, 'fake.builder',
open=fake_open)
#test non accessible builder file
fake_open.side_effect = io_error_no_perm
self.assertRaises(exceptions.PermissionError,
ring.RingBuilder.load, 'fake.builder',
open=fake_open)
#test an error other then ENOENT and ENOPERM
fake_open.side_effect = io_error_generic
self.assertRaises(IOError,
ring.RingBuilder.load, 'fake.builder',
open=fake_open)
def test_save_load(self):
rb = ring.RingBuilder(8, 3, 1)
devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1,

90
test/unit/common/test_db.py
View File

@ -32,11 +32,14 @@ from mock import patch, MagicMock
from eventlet.timeout import Timeout
import swift.common.db
from swift.common.constraints import \
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE
from swift.common.db import chexor, dict_factory, get_db_connection, \
DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists, \
GreenDBConnection, PICKLE_PROTOCOL
from swift.common.utils import normalize_timestamp, mkdirs, json, Timestamp
from swift.common.exceptions import LockTimeout
from swift.common.swob import HTTPException
from test.unit import with_tempdir
@ -655,7 +658,7 @@ class TestDatabaseBroker(unittest.TestCase):
conn.execute('CREATE TABLE test (one TEXT)')
conn.execute('CREATE TABLE test_stat (id TEXT)')
conn.execute('INSERT INTO test_stat (id) VALUES (?)',
(str(uuid4),))
(str(uuid4),))
conn.execute('INSERT INTO test (one) VALUES ("1")')
conn.commit()
stub_called = [False]
@ -1107,6 +1110,91 @@ class TestDatabaseBroker(unittest.TestCase):
[first_value, first_timestamp])
self.assert_('Second' not in broker.metadata)
@patch.object(DatabaseBroker, 'validate_metadata')
def test_validate_metadata_is_called_from_update_metadata(self, mock):
broker = self.get_replication_info_tester(metadata=True)
first_timestamp = normalize_timestamp(1)
first_value = '1'
metadata = {'First': [first_value, first_timestamp]}
broker.update_metadata(metadata, validate_metadata=True)
self.assertTrue(mock.called)
@patch.object(DatabaseBroker, 'validate_metadata')
def test_validate_metadata_is_not_called_from_update_metadata(self, mock):
broker = self.get_replication_info_tester(metadata=True)
first_timestamp = normalize_timestamp(1)
first_value = '1'
metadata = {'First': [first_value, first_timestamp]}
broker.update_metadata(metadata)
self.assertFalse(mock.called)
def test_metadata_with_max_count(self):
metadata = {}
for c in xrange(MAX_META_COUNT):
key = 'X-Account-Meta-F{0}'.format(c)
metadata[key] = ('B', normalize_timestamp(1))
key = 'X-Account-Meta-Foo'.format(c)
metadata[key] = ('', normalize_timestamp(1))
try:
DatabaseBroker.validate_metadata(metadata)
except HTTPException:
self.fail('Unexpected HTTPException')
def test_metadata_raises_exception_over_max_count(self):
metadata = {}
for c in xrange(MAX_META_COUNT + 1):
key = 'X-Account-Meta-F{0}'.format(c)
metadata[key] = ('B', normalize_timestamp(1))
message = ''
try:
DatabaseBroker.validate_metadata(metadata)
except HTTPException as e:
message = str(e)
self.assertEqual(message, '400 Bad Request')
def test_metadata_with_max_overall_size(self):
metadata = {}
metadata_value = 'v' * MAX_META_VALUE_LENGTH
size = 0
x = 0
while size < (MAX_META_OVERALL_SIZE - 4
- MAX_META_VALUE_LENGTH):
size += 4 + MAX_META_VALUE_LENGTH
metadata['X-Account-Meta-%04d' % x] = (metadata_value,
normalize_timestamp(1))
x += 1
if MAX_META_OVERALL_SIZE - size > 1:
metadata['X-Account-Meta-k'] = (
'v' * (MAX_META_OVERALL_SIZE - size - 1),
normalize_timestamp(1))
try:
DatabaseBroker.validate_metadata(metadata)
except HTTPException:
self.fail('Unexpected HTTPException')
def test_metadata_raises_exception_over_max_overall_size(self):
metadata = {}
metadata_value = 'k' * MAX_META_VALUE_LENGTH
size = 0
x = 0
while size < (MAX_META_OVERALL_SIZE - 4
- MAX_META_VALUE_LENGTH):
size += 4 + MAX_META_VALUE_LENGTH
metadata['X-Account-Meta-%04d' % x] = (metadata_value,
normalize_timestamp(1))
x += 1
if MAX_META_OVERALL_SIZE - size > 1:
metadata['X-Account-Meta-k'] = (
'v' * (MAX_META_OVERALL_SIZE - size - 1),
normalize_timestamp(1))
metadata['X-Account-Meta-k2'] = ('v', normalize_timestamp(1))
message = ''
try:
DatabaseBroker.validate_metadata(metadata)
except HTTPException as e:
message = str(e)
self.assertEqual(message, '400 Bad Request')
if __name__ == '__main__':
unittest.main()

17
test/unit/common/test_direct_client.py
View File

@ -161,13 +161,19 @@ class TestDirectClient(unittest.TestCase):
with mocked_http_conn(200, stub_headers, body) as conn:
resp_headers, resp = direct_client.direct_get_account(
self.node, self.part, self.account)
self.node, self.part, self.account, marker='marker',
prefix='prefix', delimiter='delimiter', limit=1000)
self.assertEqual(conn.method, 'GET')
self.assertEqual(conn.path, self.account_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(resp_headers, stub_headers)
self.assertEqual(json.loads(body), resp)
self.assertTrue('marker=marker' in conn.query_string)
self.assertTrue('delimiter=delimiter' in conn.query_string)
self.assertTrue('limit=1000' in conn.query_string)
self.assertTrue('prefix=prefix' in conn.query_string)
self.assertTrue('format=json' in conn.query_string)
def test_direct_client_exception(self):
stub_headers = {'X-Trans-Id': 'txb5f59485c578460f8be9e-0053478d09'}
@ -302,12 +308,19 @@ class TestDirectClient(unittest.TestCase):
with mocked_http_conn(200, headers, body) as conn:
resp_headers, resp = direct_client.direct_get_container(
self.node, self.part, self.account, self.container)
self.node, self.part, self.account, self.container,
marker='marker', prefix='prefix', delimiter='delimiter',
limit=1000)
self.assertEqual(conn.req_headers['user-agent'],
'direct-client %s' % os.getpid())
self.assertEqual(headers, resp_headers)
self.assertEqual(json.loads(body), resp)
self.assertTrue('marker=marker' in conn.query_string)
self.assertTrue('delimiter=delimiter' in conn.query_string)
self.assertTrue('limit=1000' in conn.query_string)
self.assertTrue('prefix=prefix' in conn.query_string)
self.assertTrue('format=json' in conn.query_string)
def test_direct_get_container_no_content_does_not_decode_body(self):
headers = {}

6
test/unit/common/test_manager.py
View File

@ -1476,6 +1476,7 @@ class TestManager(unittest.TestCase):
def launch(self, **kwargs):
self.called['launch'].append(kwargs)
return {}
def wait(self, **kwargs):
self.called['wait'].append(kwargs)
@ -1538,6 +1539,7 @@ class TestManager(unittest.TestCase):
def launch(self, **kwargs):
self.called['launch'].append(kwargs)
return {}
def wait(self, **kwargs):
self.called['wait'].append(kwargs)
@ -1589,6 +1591,7 @@ class TestManager(unittest.TestCase):
def launch(self, **kwargs):
self.called['launch'].append(kwargs)
return {}
def interact(self, **kwargs):
self.called['interact'].append(kwargs)
@ -1630,7 +1633,8 @@ class TestManager(unittest.TestCase):
return 0
def launch(self, **kwargs):
return self.called['launch'].append(kwargs)
self.called['launch'].append(kwargs)
return {}
orig_swift_server = manager.Server
try:

3
test/unit/common/test_utils.py
View File

@ -3863,6 +3863,7 @@ class TestAuditLocationGenerator(unittest.TestCase):
audit = lambda: list(utils.audit_location_generator(
tmpdir, "data", mount_check=False))
self.assertRaises(OSError, audit)
rmtree(tmpdir)
#Check Raise on Bad Suffix
tmpdir = mkdtemp()
@ -3881,6 +3882,7 @@ class TestAuditLocationGenerator(unittest.TestCase):
audit = lambda: list(utils.audit_location_generator(
tmpdir, "data", mount_check=False))
self.assertRaises(OSError, audit)
rmtree(tmpdir)
#Check Raise on Bad Hash
tmpdir = mkdtemp()
@ -3899,6 +3901,7 @@ class TestAuditLocationGenerator(unittest.TestCase):
audit = lambda: list(utils.audit_location_generator(
tmpdir, "data", mount_check=False))
self.assertRaises(OSError, audit)
rmtree(tmpdir)
def test_non_dir_drive(self):
with temptree([]) as tmpdir:

59
test/unit/common/test_wsgi.py
View File

@ -317,7 +317,6 @@ class TestWSGI(unittest.TestCase):
def test_run_server(self):
config = """
[DEFAULT]
eventlet_debug = yes
client_timeout = 30
max_clients = 1000
swift_dir = TEMPDIR
@ -354,7 +353,7 @@ class TestWSGI(unittest.TestCase):
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
socket=True)
_eventlet.debug.hub_exceptions.assert_called_with(True)
_eventlet.debug.hub_exceptions.assert_called_with(False)
_wsgi.server.assert_called()
args, kwargs = _wsgi.server.call_args
server_sock, server_app, server_logger = args
@ -414,7 +413,6 @@ class TestWSGI(unittest.TestCase):
""",
'proxy-server.conf.d/default.conf': """
[DEFAULT]
eventlet_debug = yes
client_timeout = 30
"""
}
@ -443,7 +441,7 @@ class TestWSGI(unittest.TestCase):
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
socket=True)
_eventlet.debug.hub_exceptions.assert_called_with(True)
_eventlet.debug.hub_exceptions.assert_called_with(False)
_wsgi.server.assert_called()
args, kwargs = _wsgi.server.call_args
server_sock, server_app, server_logger = args
@ -452,6 +450,59 @@ class TestWSGI(unittest.TestCase):
self.assert_(isinstance(server_logger, wsgi.NullLogger))
self.assert_('custom_pool' in kwargs)
def test_run_server_debug(self):
config = """
[DEFAULT]
eventlet_debug = yes
client_timeout = 30
max_clients = 1000
swift_dir = TEMPDIR
[pipeline:main]
pipeline = proxy-server
[app:proxy-server]
use = egg:swift#proxy
# while "set" values normally override default
set client_timeout = 20
# this section is not in conf during run_server
set max_clients = 10
"""
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
with mock.patch('swift.proxy.server.Application.'
'modify_wsgi_pipeline'):
with mock.patch('swift.common.wsgi.wsgi') as _wsgi:
mock_server = _wsgi.server
_wsgi.server = lambda *args, **kwargs: mock_server(
*args, **kwargs)
with mock.patch('swift.common.wsgi.eventlet') as _eventlet:
conf = wsgi.appconfig(conf_file)
logger = logging.getLogger('test')
sock = listen(('localhost', 0))
wsgi.run_server(conf, logger, sock)
self.assertEquals('HTTP/1.0',
_wsgi.HttpProtocol.default_request_version)
self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
socket=True)
_eventlet.debug.hub_exceptions.assert_called_with(True)
mock_server.assert_called()
args, kwargs = mock_server.call_args
server_sock, server_app, server_logger = args
self.assertEquals(sock, server_sock)
self.assert_(isinstance(server_app, swift.proxy.server.Application))
self.assertEquals(20, server_app.client_timeout)
self.assertEqual(server_logger, None)
self.assert_('custom_pool' in kwargs)
self.assertEquals(1000, kwargs['custom_pool'].size)
def test_appconfig_dir_ignores_hidden_files(self):
config_dir = {
'server.conf.d/01.conf': """

12
test/unit/obj/test_auditor.py
View File

@ -491,6 +491,18 @@ class TestAuditor(unittest.TestCase):
finally:
auditor.diskfile.DiskFile = was_df
@mock.patch.object(auditor.ObjectAuditor, 'run_audit')
@mock.patch('os.fork', return_value=0)
def test_with_inaccessible_object_location(self, mock_os_fork,
mock_run_audit):
# Need to ensure that any failures in run_audit do
# not prevent sys.exit() from running. Otherwise we get
# zombie processes.
e = OSError('permission denied')
mock_run_audit.side_effect = e
self.auditor = auditor.ObjectAuditor(self.conf)
self.assertRaises(SystemExit, self.auditor.fork_child, self)
def test_with_tombstone(self):
ts_file_path = self.setup_bad_zero_byte(with_ts=True)
self.assertTrue(ts_file_path.endswith('ts'))

5
test/unit/obj/test_expirer.py
View File

@ -41,6 +41,7 @@ def not_sleep(seconds):
class TestObjectExpirer(TestCase):
maxDiff = None
internal_client = None
def setUp(self):
global not_sleep
@ -54,10 +55,10 @@ class TestObjectExpirer(TestCase):
self.rcache = mkdtemp()
self.logger = FakeLogger()
def teardown(self):
def tearDown(self):
rmtree(self.rcache)
internal_client.sleep = self.old_sleep
internal_client.loadapp = self.loadapp
internal_client.loadapp = self.old_loadapp
def test_get_process_values_from_kwargs(self):
x = expirer.ObjectExpirer({})

4
test/unit/obj/test_replicator.py
View File

@ -167,9 +167,9 @@ class TestObjectReplicator(unittest.TestCase):
self.parts_1 = {}
for part in ['0', '1', '2', '3']:
self.parts[part] = os.path.join(self.objects, part)
os.mkdir(os.path.join(self.objects, part))
os.mkdir(self.parts[part])
self.parts_1[part] = os.path.join(self.objects_1, part)
os.mkdir(os.path.join(self.objects_1, part))
os.mkdir(self.parts_1[part])
_create_test_rings(self.testdir)
self.conf = dict(
swift_dir=self.testdir, devices=self.devices, mount_check='false',

13
test/unit/obj/test_server.py
View File

@ -63,8 +63,9 @@ class TestObjectController(unittest.TestCase):
"""Set up for testing swift.object.server.ObjectController"""
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = 'startcap'
self.testdir = \
os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController')
self.tmpdir = mkdtemp()
self.testdir = os.path.join(self.tmpdir,
'tmp_test_object_server_ObjectController')
conf = {'devices': self.testdir, 'mount_check': 'false'}
self.object_controller = object_server.ObjectController(
conf, logger=debug_logger())
@ -76,7 +77,7 @@ class TestObjectController(unittest.TestCase):
def tearDown(self):
"""Tear down for testing swift.object.server.ObjectController"""
rmtree(os.path.dirname(self.testdir))
rmtree(self.tmpdir)
tpool.execute = self._orig_tpool_exc
def _stage_tmp_dir(self, policy):
@ -4318,7 +4319,8 @@ class TestObjectServer(unittest.TestCase):
def setUp(self):
# dirs
self.tempdir = os.path.join(tempfile.mkdtemp(), 'tmp_test_obj_server')
self.tmpdir = tempfile.mkdtemp()
self.tempdir = os.path.join(self.tmpdir, 'tmp_test_obj_server')
self.devices = os.path.join(self.tempdir, 'srv/node')
for device in ('sda1', 'sdb1'):
@ -4335,6 +4337,9 @@ class TestObjectServer(unittest.TestCase):
self.server = spawn(wsgi.server, sock, app, utils.NullLogger())
self.port = sock.getsockname()[1]
def tearDown(self):
rmtree(self.tmpdir)
def test_not_found(self):
conn = bufferedhttp.http_connect('127.0.0.1', self.port, 'sda1', '0',
'GET', '/a/c/o')

9
test/unit/proxy/test_sysmeta.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
import unittest
import os
import shutil
from tempfile import mkdtemp
from urllib import quote
from swift.common.storage_policy import StoragePolicy, REPL_POLICY
@ -129,8 +130,9 @@ class TestObjectSysmeta(unittest.TestCase):
account_ring=FakeRing(replicas=1),
container_ring=FakeRing(replicas=1))
monkey_patch_mimetools()
self.testdir = \
os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController')
self.tmpdir = mkdtemp()
self.testdir = os.path.join(self.tmpdir,
'tmp_test_object_server_ObjectController')
mkdirs(os.path.join(self.testdir, 'sda1', 'tmp'))
conf = {'devices': self.testdir, 'mount_check': 'false'}
self.obj_ctlr = object_server.ObjectController(
@ -143,6 +145,9 @@ class TestObjectSysmeta(unittest.TestCase):
swift.proxy.controllers.base.http_connect = http_connect
swift.proxy.controllers.obj.http_connect = http_connect
def tearDown(self):
shutil.rmtree(self.tmpdir)
original_sysmeta_headers_1 = {'x-object-sysmeta-test0': 'val0',
'x-object-sysmeta-test1': 'val1'}
original_sysmeta_headers_2 = {'x-object-sysmeta-test2': 'val2'}

5
tox.ini
View File

@ -58,10 +58,11 @@ commands = python setup.py build_sphinx
# it's not a bug that we aren't using all of hacking
# H102 -> apache2 license exists
# H103 -> license is apache
# H201 -> no bare excepts # add when hacking supports noqa
# H201 -> no bare excepts (unless marked with " # noqa")
# H231 -> Check for except statements to be Python 3.x compatible
# H501 -> don't use locals() for str formatting
# H903 -> \n not \r\n
ignore = H
select = F,E,W,H102,H103,H501,H903,H231
select = F,E,W,H102,H103,H201,H231,H501,H903
exclude = .venv,.tox,dist,doc,*egg
show-source = True