openstack
/
swift
Code Issues Proposed changes

swift constraints are now settable via config 

Change previously hard-coded constants into config variables. This
allows deployers to tune their cluster more specifically based on
their needs. For example, a deployment that uses direct swift access
for public content may need to set a larger header value constraint to
allow for the full object name to be represented in the Content-
Disposition header (which browsers check to determine the name of a
downloaded object).

The new settings are set in the [swift-constraints] section of
/etc/swift/swift.conf. Comments were also added to this config file.

Cleaned up swift/common/constraints.py to pass pep8 1.3.3

Funtional tests now require constraints to be defined in /etc/test.conf or in
/etc/swift/swift.conf (in the case of running the functional tests against a
local swift cluster). To have any hope of tests passing, the defined
constraints must match the constraints on the tested cluster.

Removed a ton of "magic numbers" in both unit and functional tests.

Change-Id: Ie4588e052fd158314ddca6cd8fca9bc793311465
This commit is contained in:
John Dickinson 2012-09-05 20:49:50 -07:00
parent 7f89e50eaf
commit a2ac5efaa6
8 changed files with 231 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
etc/swift.conf-sample
View File

@ -1,3 +1,76 @@
[swift-hash]
# swift_hash_path_suffix is used as part of the hashing algorithm
# when determining data placement in the cluster. This value should
# remain secret and MUST NOT change once a cluster has been deployed.
swift_hash_path_suffix = changeme
# The swift-constraints section sets the basic constraints on data
# saved in the swift cluster.
[swift-constraints]
# max_file_size is the largest "normal" object that can be saved in
# the cluster. This is also the limit on the size of each segment of
# a "large" object when using the large object manifest support.
# This value is set in bytes. Setting it to lower than 1MiB will cause
# some tests to fail. It is STRONGLY recommended to leave this value at
# the default (5 * 2**30 + 2).
#max_file_size = 5368709122
# max_meta_name_length is the max number of bytes in the utf8 encoding
# of the name portion of a metadata header.
#max_meta_name_length = 128
# max_meta_value_length is the max number of bytes in the utf8 encoding
# of a metadata value
#max_meta_value_length = 256
# max_meta_count is the max number of metadata keys that can be stored
# on a single account, container, or object
#max_meta_count = 90
# max_meta_overall_size is the max number of bytes in the utf8 encoding
# of the metadata (keys + values)
#max_meta_overall_size = 4096
# max_object_name_length is the max number of bytes in the utf8 encoding
# of an object name
#max_object_name_length = 1024
# container_listing_limit is the default (and max) number of items
# returned for a container listing request
#container_listing_limit = 10000
# account_listing_limit is the default (and max) number of items returned
# for an account listing request
#account_listing_limit = 10000
# max_account_name_length is the max number of bytes in the utf8 encoding
# of an account name
#max_account_name_length = 256
# max_container_name_length is the max number of bytes in the utf8 encoding
# of a container name
#max_container_name_length = 256

75
swift/common/constraints.py
View File

@ -14,29 +14,48 @@
# limitations under the License.
import os
from ConfigParser import ConfigParser, NoSectionError, NoOptionError, \
RawConfigParser
from webob.exc import HTTPBadRequest, HTTPLengthRequired, \
HTTPRequestEntityTooLarge
constraints_conf = ConfigParser()
constraints_conf.read('/etc/swift/swift.conf')
def constraints_conf_int(name, default):
try:
return int(constraints_conf.get('swift-constraints', name))
except (NoSectionError, NoOptionError):
return default
#: Max file size allowed for objects
MAX_FILE_SIZE = 5 * 1024 * 1024 * 1024 + 2
MAX_FILE_SIZE = constraints_conf_int('max_file_size',
5368709122) # 5 * 1024 * 1024 * 1024 + 2
#: Max length of the name of a key for metadata
MAX_META_NAME_LENGTH = 128
MAX_META_NAME_LENGTH = constraints_conf_int('max_meta_name_length', 128)
#: Max length of the value of a key for metadata
MAX_META_VALUE_LENGTH = 256
MAX_META_VALUE_LENGTH = constraints_conf_int('max_meta_value_length', 256)
#: Max number of metadata items
MAX_META_COUNT = 90
MAX_META_COUNT = constraints_conf_int('max_meta_count', 90)
#: Max overall size of metadata
MAX_META_OVERALL_SIZE = 4096
MAX_META_OVERALL_SIZE = constraints_conf_int('max_meta_overall_size', 4096)
#: Max object name length
MAX_OBJECT_NAME_LENGTH = 1024
MAX_OBJECT_NAME_LENGTH = constraints_conf_int('max_object_name_length', 1024)
#: Max object list length of a get request for a container
CONTAINER_LISTING_LIMIT = 10000
CONTAINER_LISTING_LIMIT = constraints_conf_int('container_listing_limit',
10000)
#: Max container list length of a get request for an account
ACCOUNT_LISTING_LIMIT = 10000
MAX_ACCOUNT_NAME_LENGTH = 256
MAX_CONTAINER_NAME_LENGTH = 256
ACCOUNT_LISTING_LIMIT = constraints_conf_int('account_listing_limit', 10000)
#: Max account name length
MAX_ACCOUNT_NAME_LENGTH = constraints_conf_int('max_account_name_length', 256)
#: Max container name length
MAX_CONTAINER_NAME_LENGTH = constraints_conf_int('max_container_name_length',
256)
#: Query string format= values to their corresponding content-type values
FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json',
'xml': 'application/xml'}
@ -60,28 +79,26 @@ def check_metadata(req, target_type):
key = key[len(prefix):]
if not key:
return HTTPBadRequest(body='Metadata name cannot be empty',
request=req, content_type='text/plain')
request=req, content_type='text/plain')
meta_count += 1
meta_size += len(key) + len(value)
if len(key) > MAX_META_NAME_LENGTH:
return HTTPBadRequest(
body='Metadata name too long; max %d'
% MAX_META_NAME_LENGTH,
request=req, content_type='text/plain')
body='Metadata name too long; max %d' % MAX_META_NAME_LENGTH,
request=req, content_type='text/plain')
elif len(value) > MAX_META_VALUE_LENGTH:
return HTTPBadRequest(
body='Metadata value too long; max %d'
% MAX_META_VALUE_LENGTH,
request=req, content_type='text/plain')
body='Metadata value too long; max %d' % MAX_META_VALUE_LENGTH,
request=req, content_type='text/plain')
elif meta_count > MAX_META_COUNT:
return HTTPBadRequest(
body='Too many metadata items; max %d' % MAX_META_COUNT,
request=req, content_type='text/plain')
body='Too many metadata items; max %d' % MAX_META_COUNT,
request=req, content_type='text/plain')
elif meta_size > MAX_META_OVERALL_SIZE:
return HTTPBadRequest(
body='Total metadata too large; max %d'
% MAX_META_OVERALL_SIZE,
request=req, content_type='text/plain')
body='Total metadata too large; max %d'
% MAX_META_OVERALL_SIZE,
request=req, content_type='text/plain')
return None
@ -99,7 +116,8 @@ def check_object_creation(req, object_name):
"""
if req.content_length and req.content_length > MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(body='Your request is too large.',
request=req, content_type='text/plain')
request=req,
content_type='text/plain')
if req.content_length is None and \
req.headers.get('transfer-encoding') != 'chunked':
return HTTPLengthRequired(request=req)
@ -108,14 +126,14 @@ def check_object_creation(req, object_name):
request=req, content_type='text/plain')
if len(object_name) > MAX_OBJECT_NAME_LENGTH:
return HTTPBadRequest(body='Object name length of %d longer than %d' %
(len(object_name), MAX_OBJECT_NAME_LENGTH), request=req,
content_type='text/plain')
(len(object_name), MAX_OBJECT_NAME_LENGTH),
request=req, content_type='text/plain')
if 'Content-Type' not in req.headers:
return HTTPBadRequest(request=req, content_type='text/plain',
body='No content type')
body='No content type')
if not check_utf8(req.headers['Content-Type']):
return HTTPBadRequest(request=req, body='Invalid Content-Type',
content_type='text/plain')
content_type='text/plain')
if 'x-object-manifest' in req.headers:
value = req.headers['x-object-manifest']
container = prefix = None
@ -125,7 +143,8 @@ def check_object_creation(req, object_name):
pass
if not container or not prefix or '?' in value or '&' in value or \
prefix[0] == '/':
return HTTPBadRequest(request=req,
return HTTPBadRequest(
request=req,
body='X-Object-Manifest must in the format container/prefix')
return check_metadata(req, 'object')

0
test/functional/swift.py → test/functional/swift_test_client.py
View File

63
test/functional/tests.py
View File

@ -23,11 +23,55 @@ import threading
import uuid
import unittest
from nose import SkipTest
from ConfigParser import ConfigParser
from test import get_config
from test.functional.swift import Account, Connection, File, ResponseError
from test.functional.swift_test_client import Account, Connection, File, \
ResponseError
from swift.common.constraints import MAX_FILE_SIZE, MAX_META_NAME_LENGTH, \
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \
MAX_OBJECT_NAME_LENGTH, CONTAINER_LISTING_LIMIT, ACCOUNT_LISTING_LIMIT, \
MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH
default_constraints = dict((
('max_file_size', MAX_FILE_SIZE),
('max_meta_name_length', MAX_META_NAME_LENGTH),
('max_meta_value_length', MAX_META_VALUE_LENGTH),
('max_meta_count', MAX_META_COUNT),
('max_meta_overall_size', MAX_META_OVERALL_SIZE),
('max_object_name_length', MAX_OBJECT_NAME_LENGTH),
('container_listing_limit', CONTAINER_LISTING_LIMIT),
('account_listing_limit', ACCOUNT_LISTING_LIMIT),
('max_account_name_length', MAX_ACCOUNT_NAME_LENGTH),
('max_container_name_length', MAX_CONTAINER_NAME_LENGTH)))
constraints_conf = ConfigParser()
conf_exists = constraints_conf.read('/etc/swift/swift.conf')
# Constraints are set first from the test config, then from
# /etc/swift/swift.conf if it exists. If swift.conf doesn't exist,
# then limit test coverage. This allows SAIO tests to work fine but
# requires remote funtional testing to know something about the cluster
# that is being tested.
config = get_config('func_test')
for k in default_constraints:
if k in config:
# prefer what's in test.conf
config[k] = int(config[k])
elif conf_exists:
# swift.conf exists, so use what's defined there (or swift defaults)
# This normally happens when the test is running locally to the cluster
# as in a SAIO.
config[k] = default_constraints[k]
else:
# .functests don't know what the constraints of the tested cluster are,
# so the tests can't reliably pass or fail. Therefore, skip those
# tests.
config[k] = '%s constraint is not defined' % k
def load_constraint(name):
c = config[name]
if not isinstance(c, int):
raise SkipTest(c)
return c
locale.setlocale(locale.LC_COLLATE, config.get('collate', 'C'))
@ -231,8 +275,7 @@ class TestAccount(Base):
'application/xml; charset=utf-8')
def testListingLimit(self):
limit = 10000
limit = load_constraint('account_listing_limit')
for l in (1, 100, limit / 2, limit - 1, limit, limit + 1, limit * 2):
p = {'limit': l}
@ -367,7 +410,7 @@ class TestContainer(Base):
set_up = False
def testContainerNameLimit(self):
limit = 256
limit = load_constraint('max_container_name_length')
for l in (limit - 100, limit - 10, limit - 1, limit,
limit + 1, limit + 10, limit + 100):
@ -412,6 +455,7 @@ class TestContainer(Base):
self.assert_(cont.files(parms={'prefix': f}) == [f])
def testPrefixAndLimit(self):
load_constraint('container_listing_limit')
cont = self.env.account.container(Utils.create_name())
self.assert_(cont.create())
@ -967,7 +1011,7 @@ class TestFile(Base):
self.assert_status(404)
def testNameLimit(self):
limit = 1024
limit = load_constraint('max_object_name_length')
for l in (1, 10, limit / 2, limit - 1, limit, limit + 1, limit * 2):
file = self.env.container.file('a' * l)
@ -1014,11 +1058,11 @@ class TestFile(Base):
self.assert_status(400)
def testMetadataNumberLimit(self):
number_limit = 90
number_limit = load_constraint('max_meta_count')
size_limit = load_constraint('max_meta_overall_size')
for i in (number_limit - 10, number_limit - 1, number_limit,
number_limit + 1, number_limit + 10, number_limit + 100):
size_limit = 4096
j = size_limit / (i * 2)
@ -1120,7 +1164,7 @@ class TestFile(Base):
self.assert_(file.read(hdrs={'Range': r}) == data[0:1000])
def testFileSizeLimit(self):
limit = 5 * 2 ** 30 + 2
limit = load_constraint('max_file_size')
tsecs = 3
for i in (limit - 100, limit - 10, limit - 1, limit, limit + 1,
@ -1176,7 +1220,8 @@ class TestFile(Base):
self.assert_status(200)
def testMetadataLengthLimits(self):
key_limit, value_limit = 128, 256
key_limit = load_constraint('max_meta_name_length')
value_limit = load_constraint('max_meta_value_length')
lengths = [[key_limit, value_limit], [key_limit, value_limit + 1],
[key_limit + 1, value_limit], [key_limit, 0],
[key_limit, value_limit * 10],

15
test/sample.conf
View File

@ -19,6 +19,21 @@ password2 = testing2
username3 = tester3
password3 = testing3
# Default constraints if not defined here, the test runner will try
# to set them from /etc/swift/swift.conf. If that file isn't found,
# the test runner will skip tests that depend on these values.
# Note that the cluster must have "sane" values for the test suite to pass.
#max_file_size = 5368709122
#max_meta_name_length = 128
#max_meta_value_length = 256
#max_meta_count = 90
#max_meta_overall_size = 4096
#max_object_name_length = 1024
#container_listing_limit = 10000
#account_listing_limit = 10000
#max_account_name_length = 256
#max_container_name_length = 256
collate = C
[unit_test]

9
test/unit/common/test_constraints.py
View File

@ -84,8 +84,13 @@ class TestConstraints(unittest.TestCase):
x += 1
self.assertEquals(constraints.check_metadata(Request.blank('/',
headers=headers), 'object'), None)
headers['X-Object-Meta-9999%s' %
('a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \
# add two more headers in case adding just one falls exactly on the
# limit (eg one header adds 1024 and the limit is 2048)
headers['X-Object-Meta-%04d%s' %
(x, 'a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \
'v' * constraints.MAX_META_VALUE_LENGTH
headers['X-Object-Meta-%04d%s' %
(x + 1, 'a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \
'v' * constraints.MAX_META_VALUE_LENGTH
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
headers=headers), 'object'), HTTPBadRequest))

6
test/unit/obj/test_server.py
View File

@ -35,6 +35,7 @@ from swift.common import utils
from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \
NullLogger, storage_directory
from swift.common.exceptions import DiskFileNotExist
from swift.common import constraints
from eventlet import tpool
@ -1389,7 +1390,8 @@ class TestObjectController(unittest.TestCase):
def test_max_object_name_length(self):
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/' + ('1' * 1024),
max_name_len = constraints.MAX_OBJECT_NAME_LENGTH
req = Request.blank('/sda1/p/a/c/' + ('1' * max_name_len),
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
'Content-Length': '4',
@ -1397,7 +1399,7 @@ class TestObjectController(unittest.TestCase):
req.body = 'DATA'
resp = self.object_controller.PUT(req)
self.assertEquals(resp.status_int, 201)
req = Request.blank('/sda1/p/a/c/' + ('2' * 1025),
req = Request.blank('/sda1/p/a/c/' + ('2' * (max_name_len + 1)),
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
'Content-Length': '4',

50
test/unit/proxy/test_server.py
View File

@ -46,7 +46,8 @@ from swift.obj import server as object_server
from swift.common import ring
from swift.common.exceptions import ChunkReadTimeout
from swift.common.constraints import MAX_META_NAME_LENGTH, \
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, MAX_FILE_SIZE
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \
MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH
from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
from swift.common.wsgi import monkey_patch_mimetools
from swift.proxy.controllers.obj import SegmentedIterable
@ -1037,47 +1038,50 @@ class TestObjectController(unittest.TestCase):
def test_POST_meta_val_len(self):
with save_globals():
limit = MAX_META_VALUE_LENGTH
self.app.object_post_as_copy = False
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
set_http_connect(200, 200, 202, 202, 202)
# acct cont obj obj obj
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * 256})
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * limit})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
set_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * 257})
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * (limit + 1)})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_POST_as_copy_meta_val_len(self):
with save_globals():
limit = MAX_META_VALUE_LENGTH
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
set_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
# acct cont objc objc objc obj obj obj
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * 256})
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * limit})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
set_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * 257})
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * (limit + 1)})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_POST_meta_key_len(self):
with save_globals():
limit = MAX_META_NAME_LENGTH
self.app.object_post_as_copy = False
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
@ -1085,44 +1089,46 @@ class TestObjectController(unittest.TestCase):
# acct cont obj obj obj
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-' + 'x' * 128): 'x'})
('X-Object-Meta-' + 'x' * limit): 'x'})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
set_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-' + 'x' * 129): 'x'})
('X-Object-Meta-' + 'x' * (limit + 1)): 'x'})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_POST_as_copy_meta_key_len(self):
with save_globals():
limit = MAX_META_NAME_LENGTH
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
set_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
# acct cont objc objc objc obj obj obj
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-' + 'x' * 128): 'x'})
('X-Object-Meta-' + 'x' * limit): 'x'})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
set_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-' + 'x' * 129): 'x'})
('X-Object-Meta-' + 'x' * (limit + 1)): 'x'})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_POST_meta_count(self):
with save_globals():
limit = MAX_META_COUNT
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
headers = dict(
(('X-Object-Meta-' + str(i), 'a') for i in xrange(91)))
(('X-Object-Meta-' + str(i), 'a') for i in xrange(limit + 1)))
headers.update({'Content-Type': 'foo/bar'})
set_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers=headers)
@ -1132,10 +1138,13 @@ class TestObjectController(unittest.TestCase):
def test_POST_meta_size(self):
with save_globals():
limit = MAX_META_OVERALL_SIZE
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
count = limit / 256 # enough to cause the limit to be reched
headers = dict(
(('X-Object-Meta-' + str(i), 'a' * 256) for i in xrange(1000)))
(('X-Object-Meta-' + str(i), 'a' * 256)
for i in xrange(count + 1)))
headers.update({'Content-Type': 'foo/bar'})
set_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers=headers)
@ -3348,13 +3357,14 @@ class TestContainerController(unittest.TestCase):
def test_PUT_max_container_name_length(self):
with save_globals():
limit = MAX_CONTAINER_NAME_LENGTH
controller = proxy_server.ContainerController(self.app, 'account',
'1' * 256)
'1' * limit)
self.assert_status_map(controller.PUT,
(200, 200, 200, 201, 201, 201), 201,
missing_container=True)
controller = proxy_server.ContainerController(self.app, 'account',
'2' * 257)
'2' * (limit + 1))
self.assert_status_map(controller.PUT, (201, 201, 201), 400,
missing_container=True)
@ -3901,9 +3911,11 @@ class TestAccountController(unittest.TestCase):
def test_PUT_max_account_name_length(self):
with save_globals():
self.app.allow_account_management = True
controller = proxy_server.AccountController(self.app, '1' * 256)
limit = MAX_ACCOUNT_NAME_LENGTH
controller = proxy_server.AccountController(self.app, '1' * limit)
self.assert_status_map(controller.PUT, (201, 201, 201), 201)
controller = proxy_server.AccountController(self.app, '2' * 257)
controller = proxy_server.AccountController(
self.app, '2' * (limit + 1))
self.assert_status_map(controller.PUT, (201, 201, 201), 400)
def test_PUT_connect_exceptions(self):