Expose account/container metadata facility to external API
This commit is contained in:
parent
b1b126e461
commit
e8d3f260b9
|
@ -115,7 +115,7 @@ class AccountController(object):
|
||||||
for key, value in req.headers.iteritems()
|
for key, value in req.headers.iteritems()
|
||||||
if key.lower().startswith('x-account-meta-'))
|
if key.lower().startswith('x-account-meta-'))
|
||||||
if metadata:
|
if metadata:
|
||||||
broker.metadata = metadata
|
broker.update_metadata(metadata)
|
||||||
if created:
|
if created:
|
||||||
return HTTPCreated(request=req)
|
return HTTPCreated(request=req)
|
||||||
else:
|
else:
|
||||||
|
@ -290,7 +290,7 @@ class AccountController(object):
|
||||||
for key, value in req.headers.iteritems()
|
for key, value in req.headers.iteritems()
|
||||||
if key.lower().startswith('x-account-meta-'))
|
if key.lower().startswith('x-account-meta-'))
|
||||||
if metadata:
|
if metadata:
|
||||||
broker.metadata = metadata
|
broker.update_metadata(metadata)
|
||||||
return HTTPNoContent(request=req)
|
return HTTPNoContent(request=req)
|
||||||
|
|
||||||
def __call__(self, env, start_response):
|
def __call__(self, env, start_response):
|
||||||
|
|
|
@ -38,19 +38,22 @@ CONTAINER_LISTING_LIMIT = 10000
|
||||||
ACCOUNT_LISTING_LIMIT = 10000
|
ACCOUNT_LISTING_LIMIT = 10000
|
||||||
|
|
||||||
|
|
||||||
def check_metadata(req):
|
def check_metadata(req, target_type):
|
||||||
"""
|
"""
|
||||||
Check metadata sent for objects in the request headers.
|
Check metadata sent in the request headers.
|
||||||
|
|
||||||
:param req: request object
|
:param req: request object
|
||||||
|
:param target_type: str: one of: object, container, or account: indicates
|
||||||
|
which type the target storage for the metadata is
|
||||||
:raises HTTPBadRequest: bad metadata
|
:raises HTTPBadRequest: bad metadata
|
||||||
"""
|
"""
|
||||||
|
prefix = 'x-%s-meta-' % target_type.lower()
|
||||||
meta_count = 0
|
meta_count = 0
|
||||||
meta_size = 0
|
meta_size = 0
|
||||||
for key, value in req.headers.iteritems():
|
for key, value in req.headers.iteritems():
|
||||||
if not key.lower().startswith('x-object-meta-'):
|
if not key.lower().startswith(prefix):
|
||||||
continue
|
continue
|
||||||
key = key[len('x-object-meta-'):]
|
key = key[len(prefix):]
|
||||||
if not key:
|
if not key:
|
||||||
return HTTPBadRequest(body='Metadata name cannot be empty',
|
return HTTPBadRequest(body='Metadata name cannot be empty',
|
||||||
request=req, content_type='text/plain')
|
request=req, content_type='text/plain')
|
||||||
|
@ -106,7 +109,7 @@ def check_object_creation(req, object_name):
|
||||||
if not check_xml_encodable(req.headers['Content-Type']):
|
if not check_xml_encodable(req.headers['Content-Type']):
|
||||||
return HTTPBadRequest(request=req, body='Invalid Content-Type',
|
return HTTPBadRequest(request=req, body='Invalid Content-Type',
|
||||||
content_type='text/plain')
|
content_type='text/plain')
|
||||||
return check_metadata(req)
|
return check_metadata(req, 'object')
|
||||||
|
|
||||||
|
|
||||||
def check_mount(root, drive):
|
def check_mount(root, drive):
|
||||||
|
|
|
@ -503,8 +503,7 @@ class DatabaseBroker(object):
|
||||||
metadata = {}
|
metadata = {}
|
||||||
return metadata
|
return metadata
|
||||||
|
|
||||||
@metadata.setter
|
def update_metadata(self, metadata_updates):
|
||||||
def metadata(self, new_metadata):
|
|
||||||
"""
|
"""
|
||||||
Updates the metadata dict for the database. The metadata dict values
|
Updates the metadata dict for the database. The metadata dict values
|
||||||
are tuples of (value, timestamp) where the timestamp indicates when
|
are tuples of (value, timestamp) where the timestamp indicates when
|
||||||
|
@ -514,8 +513,8 @@ class DatabaseBroker(object):
|
||||||
:func:reclaim
|
:func:reclaim
|
||||||
"""
|
"""
|
||||||
old_metadata = self.metadata
|
old_metadata = self.metadata
|
||||||
if set(new_metadata).issubset(set(old_metadata)):
|
if set(metadata_updates).issubset(set(old_metadata)):
|
||||||
for key, (value, timestamp) in new_metadata.iteritems():
|
for key, (value, timestamp) in metadata_updates.iteritems():
|
||||||
if timestamp > old_metadata[key][1]:
|
if timestamp > old_metadata[key][1]:
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
|
@ -532,7 +531,7 @@ class DatabaseBroker(object):
|
||||||
ALTER TABLE %s_stat
|
ALTER TABLE %s_stat
|
||||||
ADD COLUMN metadata TEXT DEFAULT '' """ % self.db_type)
|
ADD COLUMN metadata TEXT DEFAULT '' """ % self.db_type)
|
||||||
md = {}
|
md = {}
|
||||||
for key, value_timestamp in new_metadata.iteritems():
|
for key, value_timestamp in metadata_updates.iteritems():
|
||||||
value, timestamp = value_timestamp
|
value, timestamp = value_timestamp
|
||||||
if key not in md or timestamp > md[key][1]:
|
if key not in md or timestamp > md[key][1]:
|
||||||
md[key] = value_timestamp
|
md[key] = value_timestamp
|
||||||
|
|
|
@ -481,7 +481,7 @@ class ReplicatorRpc(object):
|
||||||
return HTTPNotFound()
|
return HTTPNotFound()
|
||||||
raise
|
raise
|
||||||
if metadata:
|
if metadata:
|
||||||
broker.metadata = simplejson.loads(metadata)
|
broker.update_metadata(simplejson.loads(metadata))
|
||||||
if info['put_timestamp'] != put_timestamp or \
|
if info['put_timestamp'] != put_timestamp or \
|
||||||
info['created_at'] != created_at or \
|
info['created_at'] != created_at or \
|
||||||
info['delete_timestamp'] != delete_timestamp:
|
info['delete_timestamp'] != delete_timestamp:
|
||||||
|
|
|
@ -197,7 +197,7 @@ class ContainerController(object):
|
||||||
for key, value in req.headers.iteritems()
|
for key, value in req.headers.iteritems()
|
||||||
if key.lower().startswith('x-container-meta-'))
|
if key.lower().startswith('x-container-meta-'))
|
||||||
if metadata:
|
if metadata:
|
||||||
broker.metadata = metadata
|
broker.update_metadata(metadata)
|
||||||
resp = self.account_update(req, account, container, broker)
|
resp = self.account_update(req, account, container, broker)
|
||||||
if resp:
|
if resp:
|
||||||
return resp
|
return resp
|
||||||
|
@ -378,7 +378,7 @@ class ContainerController(object):
|
||||||
for key, value in req.headers.iteritems()
|
for key, value in req.headers.iteritems()
|
||||||
if key.lower().startswith('x-container-meta-'))
|
if key.lower().startswith('x-container-meta-'))
|
||||||
if metadata:
|
if metadata:
|
||||||
broker.metadata = metadata
|
broker.update_metadata(metadata)
|
||||||
return HTTPNoContent(request=req)
|
return HTTPNoContent(request=req)
|
||||||
|
|
||||||
def __call__(self, env, start_response):
|
def __call__(self, env, start_response):
|
||||||
|
|
|
@ -483,7 +483,7 @@ class ObjectController(Controller):
|
||||||
@public
|
@public
|
||||||
def POST(self, req):
|
def POST(self, req):
|
||||||
"""HTTP POST request handler."""
|
"""HTTP POST request handler."""
|
||||||
error_response = check_metadata(req)
|
error_response = check_metadata(req, 'object')
|
||||||
if error_response:
|
if error_response:
|
||||||
return error_response
|
return error_response
|
||||||
container_partition, containers = \
|
container_partition, containers = \
|
||||||
|
@ -789,6 +789,9 @@ class ContainerController(Controller):
|
||||||
@public
|
@public
|
||||||
def PUT(self, req):
|
def PUT(self, req):
|
||||||
"""HTTP PUT request handler."""
|
"""HTTP PUT request handler."""
|
||||||
|
error_response = check_metadata(req, 'container')
|
||||||
|
if error_response:
|
||||||
|
return error_response
|
||||||
if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH:
|
if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH:
|
||||||
resp = HTTPBadRequest(request=req)
|
resp = HTTPBadRequest(request=req)
|
||||||
resp.body = 'Container name length of %d longer than %d' % \
|
resp.body = 'Container name length of %d longer than %d' % \
|
||||||
|
@ -803,6 +806,8 @@ class ContainerController(Controller):
|
||||||
self.account_name, self.container_name)
|
self.account_name, self.container_name)
|
||||||
headers = {'X-Timestamp': normalize_timestamp(time.time()),
|
headers = {'X-Timestamp': normalize_timestamp(time.time()),
|
||||||
'x-cf-trans-id': self.trans_id}
|
'x-cf-trans-id': self.trans_id}
|
||||||
|
headers.update(value for value in req.headers.iteritems()
|
||||||
|
if value[0].lower().startswith('x-container-meta-'))
|
||||||
statuses = []
|
statuses = []
|
||||||
reasons = []
|
reasons = []
|
||||||
bodies = []
|
bodies = []
|
||||||
|
@ -845,6 +850,56 @@ class ContainerController(Controller):
|
||||||
return self.best_response(req, statuses, reasons, bodies,
|
return self.best_response(req, statuses, reasons, bodies,
|
||||||
'Container PUT')
|
'Container PUT')
|
||||||
|
|
||||||
|
@public
|
||||||
|
def POST(self, req):
|
||||||
|
"""HTTP POST request handler."""
|
||||||
|
error_response = check_metadata(req, 'container')
|
||||||
|
if error_response:
|
||||||
|
return error_response
|
||||||
|
account_partition, accounts = self.account_info(self.account_name)
|
||||||
|
if not accounts:
|
||||||
|
return HTTPNotFound(request=req)
|
||||||
|
container_partition, containers = self.app.container_ring.get_nodes(
|
||||||
|
self.account_name, self.container_name)
|
||||||
|
headers = {'X-Timestamp': normalize_timestamp(time.time()),
|
||||||
|
'x-cf-trans-id': self.trans_id}
|
||||||
|
headers.update(value for value in req.headers.iteritems()
|
||||||
|
if value[0].lower().startswith('x-container-meta-'))
|
||||||
|
statuses = []
|
||||||
|
reasons = []
|
||||||
|
bodies = []
|
||||||
|
for node in self.iter_nodes(container_partition, containers,
|
||||||
|
self.app.container_ring):
|
||||||
|
if self.error_limited(node):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
with ConnectionTimeout(self.app.conn_timeout):
|
||||||
|
conn = http_connect(node['ip'], node['port'],
|
||||||
|
node['device'], container_partition, 'POST',
|
||||||
|
req.path_info, headers)
|
||||||
|
with Timeout(self.app.node_timeout):
|
||||||
|
source = conn.getresponse()
|
||||||
|
body = source.read()
|
||||||
|
if 200 <= source.status < 300 \
|
||||||
|
or 400 <= source.status < 500:
|
||||||
|
statuses.append(source.status)
|
||||||
|
reasons.append(source.reason)
|
||||||
|
bodies.append(body)
|
||||||
|
elif source.status == 507:
|
||||||
|
self.error_limit(node)
|
||||||
|
except:
|
||||||
|
self.exception_occurred(node, 'Container',
|
||||||
|
'Trying to POST %s' % req.path)
|
||||||
|
if len(statuses) >= len(containers):
|
||||||
|
break
|
||||||
|
while len(statuses) < len(containers):
|
||||||
|
statuses.append(503)
|
||||||
|
reasons.append('')
|
||||||
|
bodies.append('')
|
||||||
|
self.app.memcache.delete('container%s' % req.path_info.rstrip('/'))
|
||||||
|
return self.best_response(req, statuses, reasons, bodies,
|
||||||
|
'Container POST')
|
||||||
|
|
||||||
@public
|
@public
|
||||||
def DELETE(self, req):
|
def DELETE(self, req):
|
||||||
"""HTTP DELETE request handler."""
|
"""HTTP DELETE request handler."""
|
||||||
|
@ -926,6 +981,53 @@ class AccountController(Controller):
|
||||||
return self.GETorHEAD_base(req, 'Account', partition, nodes,
|
return self.GETorHEAD_base(req, 'Account', partition, nodes,
|
||||||
req.path_info.rstrip('/'), self.app.account_ring.replica_count)
|
req.path_info.rstrip('/'), self.app.account_ring.replica_count)
|
||||||
|
|
||||||
|
@public
|
||||||
|
def POST(self, req):
|
||||||
|
"""HTTP POST request handler."""
|
||||||
|
error_response = check_metadata(req, 'account')
|
||||||
|
if error_response:
|
||||||
|
return error_response
|
||||||
|
account_partition, accounts = \
|
||||||
|
self.app.account_ring.get_nodes( self.account_name)
|
||||||
|
headers = {'X-Timestamp': normalize_timestamp(time.time()),
|
||||||
|
'X-CF-Trans-Id': self.trans_id}
|
||||||
|
headers.update(value for value in req.headers.iteritems()
|
||||||
|
if value[0].lower().startswith('x-account-meta-'))
|
||||||
|
statuses = []
|
||||||
|
reasons = []
|
||||||
|
bodies = []
|
||||||
|
for node in self.iter_nodes(account_partition, accounts,
|
||||||
|
self.app.account_ring):
|
||||||
|
if self.error_limited(node):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
with ConnectionTimeout(self.app.conn_timeout):
|
||||||
|
conn = http_connect(node['ip'], node['port'],
|
||||||
|
node['device'], account_partition, 'POST',
|
||||||
|
req.path_info, headers)
|
||||||
|
with Timeout(self.app.node_timeout):
|
||||||
|
source = conn.getresponse()
|
||||||
|
body = source.read()
|
||||||
|
if 200 <= source.status < 300 \
|
||||||
|
or 400 <= source.status < 500:
|
||||||
|
statuses.append(source.status)
|
||||||
|
reasons.append(source.reason)
|
||||||
|
bodies.append(body)
|
||||||
|
elif source.status == 507:
|
||||||
|
self.error_limit(node)
|
||||||
|
except:
|
||||||
|
self.exception_occurred(node, 'Account',
|
||||||
|
'Trying to POST %s' % req.path)
|
||||||
|
if len(statuses) >= len(accounts):
|
||||||
|
break
|
||||||
|
while len(statuses) < len(accounts):
|
||||||
|
statuses.append(503)
|
||||||
|
reasons.append('')
|
||||||
|
bodies.append('')
|
||||||
|
self.app.memcache.delete('account%s' % req.path_info.rstrip('/'))
|
||||||
|
return self.best_response(req, statuses, reasons, bodies,
|
||||||
|
'Account POST')
|
||||||
|
|
||||||
|
|
||||||
class BaseApplication(object):
|
class BaseApplication(object):
|
||||||
"""Base WSGI application for the proxy server"""
|
"""Base WSGI application for the proxy server"""
|
||||||
|
|
|
@ -207,6 +207,17 @@ class TestAccountController(unittest.TestCase):
|
||||||
resp = self.controller.GET(req)
|
resp = self.controller.GET(req)
|
||||||
self.assertEquals(resp.status_int, 204)
|
self.assertEquals(resp.status_int, 204)
|
||||||
self.assertEquals(resp.headers.get('x-account-meta-test'), 'Value')
|
self.assertEquals(resp.headers.get('x-account-meta-test'), 'Value')
|
||||||
|
# Set another metadata header, ensuring old one doesn't disappear
|
||||||
|
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
|
||||||
|
headers={'X-Timestamp': normalize_timestamp(1),
|
||||||
|
'X-Account-Meta-Test2': 'Value2'})
|
||||||
|
resp = self.controller.POST(req)
|
||||||
|
self.assertEquals(resp.status_int, 204)
|
||||||
|
req = Request.blank('/sda1/p/a')
|
||||||
|
resp = self.controller.GET(req)
|
||||||
|
self.assertEquals(resp.status_int, 204)
|
||||||
|
self.assertEquals(resp.headers.get('x-account-meta-test'), 'Value')
|
||||||
|
self.assertEquals(resp.headers.get('x-account-meta-test2'), 'Value2')
|
||||||
# Update metadata header
|
# Update metadata header
|
||||||
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
|
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
|
||||||
headers={'X-Timestamp': normalize_timestamp(3),
|
headers={'X-Timestamp': normalize_timestamp(3),
|
||||||
|
|
|
@ -27,47 +27,47 @@ class TestConstraints(unittest.TestCase):
|
||||||
def test_check_metadata_empty(self):
|
def test_check_metadata_empty(self):
|
||||||
headers = {}
|
headers = {}
|
||||||
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
||||||
headers=headers)), None)
|
headers=headers), 'object'), None)
|
||||||
|
|
||||||
def test_check_metadata_good(self):
|
def test_check_metadata_good(self):
|
||||||
headers = {'X-Object-Meta-Name': 'Value'}
|
headers = {'X-Object-Meta-Name': 'Value'}
|
||||||
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
||||||
headers=headers)), None)
|
headers=headers), 'object'), None)
|
||||||
|
|
||||||
def test_check_metadata_empty_name(self):
|
def test_check_metadata_empty_name(self):
|
||||||
headers = {'X-Object-Meta-': 'Value'}
|
headers = {'X-Object-Meta-': 'Value'}
|
||||||
self.assert_(constraints.check_metadata(Request.blank('/',
|
self.assert_(constraints.check_metadata(Request.blank('/',
|
||||||
headers=headers)), HTTPBadRequest)
|
headers=headers), 'object'), HTTPBadRequest)
|
||||||
|
|
||||||
def test_check_metadata_name_length(self):
|
def test_check_metadata_name_length(self):
|
||||||
name = 'a' * constraints.MAX_META_NAME_LENGTH
|
name = 'a' * constraints.MAX_META_NAME_LENGTH
|
||||||
headers = {'X-Object-Meta-%s' % name: 'v'}
|
headers = {'X-Object-Meta-%s' % name: 'v'}
|
||||||
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
||||||
headers=headers)), None)
|
headers=headers), 'object'), None)
|
||||||
name = 'a' * (constraints.MAX_META_NAME_LENGTH + 1)
|
name = 'a' * (constraints.MAX_META_NAME_LENGTH + 1)
|
||||||
headers = {'X-Object-Meta-%s' % name: 'v'}
|
headers = {'X-Object-Meta-%s' % name: 'v'}
|
||||||
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
|
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
|
||||||
headers=headers)), HTTPBadRequest))
|
headers=headers), 'object'), HTTPBadRequest))
|
||||||
|
|
||||||
def test_check_metadata_value_length(self):
|
def test_check_metadata_value_length(self):
|
||||||
value = 'a' * constraints.MAX_META_VALUE_LENGTH
|
value = 'a' * constraints.MAX_META_VALUE_LENGTH
|
||||||
headers = {'X-Object-Meta-Name': value}
|
headers = {'X-Object-Meta-Name': value}
|
||||||
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
||||||
headers=headers)), None)
|
headers=headers), 'object'), None)
|
||||||
value = 'a' * (constraints.MAX_META_VALUE_LENGTH + 1)
|
value = 'a' * (constraints.MAX_META_VALUE_LENGTH + 1)
|
||||||
headers = {'X-Object-Meta-Name': value}
|
headers = {'X-Object-Meta-Name': value}
|
||||||
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
|
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
|
||||||
headers=headers)), HTTPBadRequest))
|
headers=headers), 'object'), HTTPBadRequest))
|
||||||
|
|
||||||
def test_check_metadata_count(self):
|
def test_check_metadata_count(self):
|
||||||
headers = {}
|
headers = {}
|
||||||
for x in xrange(constraints.MAX_META_COUNT):
|
for x in xrange(constraints.MAX_META_COUNT):
|
||||||
headers['X-Object-Meta-%d' % x] = 'v'
|
headers['X-Object-Meta-%d' % x] = 'v'
|
||||||
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
||||||
headers=headers)), None)
|
headers=headers), 'object'), None)
|
||||||
headers['X-Object-Meta-Too-Many'] = 'v'
|
headers['X-Object-Meta-Too-Many'] = 'v'
|
||||||
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
|
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
|
||||||
headers=headers)), HTTPBadRequest))
|
headers=headers), 'object'), HTTPBadRequest))
|
||||||
|
|
||||||
def test_check_metadata_size(self):
|
def test_check_metadata_size(self):
|
||||||
headers = {}
|
headers = {}
|
||||||
|
@ -82,12 +82,12 @@ class TestConstraints(unittest.TestCase):
|
||||||
size += chunk
|
size += chunk
|
||||||
x += 1
|
x += 1
|
||||||
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
self.assertEquals(constraints.check_metadata(Request.blank('/',
|
||||||
headers=headers)), None)
|
headers=headers), 'object'), None)
|
||||||
headers['X-Object-Meta-9999%s' %
|
headers['X-Object-Meta-9999%s' %
|
||||||
('a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \
|
('a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \
|
||||||
'v' * constraints.MAX_META_VALUE_LENGTH
|
'v' * constraints.MAX_META_VALUE_LENGTH
|
||||||
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
|
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
|
||||||
headers=headers)), HTTPBadRequest))
|
headers=headers), 'object'), HTTPBadRequest))
|
||||||
|
|
||||||
def test_check_object_creation_content_length(self):
|
def test_check_object_creation_content_length(self):
|
||||||
headers = {'Content-Length': str(constraints.MAX_FILE_SIZE),
|
headers = {'Content-Length': str(constraints.MAX_FILE_SIZE),
|
||||||
|
|
|
@ -465,14 +465,14 @@ class TestDatabaseBroker(unittest.TestCase):
|
||||||
# Add our first item
|
# Add our first item
|
||||||
first_timestamp = normalize_timestamp(1)
|
first_timestamp = normalize_timestamp(1)
|
||||||
first_value = '1'
|
first_value = '1'
|
||||||
broker.metadata = {'First': [first_value, first_timestamp]}
|
broker.update_metadata({'First': [first_value, first_timestamp]})
|
||||||
self.assert_('First' in broker.metadata)
|
self.assert_('First' in broker.metadata)
|
||||||
self.assertEquals(broker.metadata['First'],
|
self.assertEquals(broker.metadata['First'],
|
||||||
[first_value, first_timestamp])
|
[first_value, first_timestamp])
|
||||||
# Add our second item
|
# Add our second item
|
||||||
second_timestamp = normalize_timestamp(2)
|
second_timestamp = normalize_timestamp(2)
|
||||||
second_value = '2'
|
second_value = '2'
|
||||||
broker.metadata = {'Second': [second_value, second_timestamp]}
|
broker.update_metadata({'Second': [second_value, second_timestamp]})
|
||||||
self.assert_('First' in broker.metadata)
|
self.assert_('First' in broker.metadata)
|
||||||
self.assertEquals(broker.metadata['First'],
|
self.assertEquals(broker.metadata['First'],
|
||||||
[first_value, first_timestamp])
|
[first_value, first_timestamp])
|
||||||
|
@ -482,7 +482,7 @@ class TestDatabaseBroker(unittest.TestCase):
|
||||||
# Update our first item
|
# Update our first item
|
||||||
first_timestamp = normalize_timestamp(3)
|
first_timestamp = normalize_timestamp(3)
|
||||||
first_value = '1b'
|
first_value = '1b'
|
||||||
broker.metadata = {'First': [first_value, first_timestamp]}
|
broker.update_metadata({'First': [first_value, first_timestamp]})
|
||||||
self.assert_('First' in broker.metadata)
|
self.assert_('First' in broker.metadata)
|
||||||
self.assertEquals(broker.metadata['First'],
|
self.assertEquals(broker.metadata['First'],
|
||||||
[first_value, first_timestamp])
|
[first_value, first_timestamp])
|
||||||
|
@ -492,7 +492,7 @@ class TestDatabaseBroker(unittest.TestCase):
|
||||||
# Delete our second item (by setting to empty string)
|
# Delete our second item (by setting to empty string)
|
||||||
second_timestamp = normalize_timestamp(4)
|
second_timestamp = normalize_timestamp(4)
|
||||||
second_value = ''
|
second_value = ''
|
||||||
broker.metadata = {'Second': [second_value, second_timestamp]}
|
broker.update_metadata({'Second': [second_value, second_timestamp]})
|
||||||
self.assert_('First' in broker.metadata)
|
self.assert_('First' in broker.metadata)
|
||||||
self.assertEquals(broker.metadata['First'],
|
self.assertEquals(broker.metadata['First'],
|
||||||
[first_value, first_timestamp])
|
[first_value, first_timestamp])
|
||||||
|
|
|
@ -104,6 +104,17 @@ class TestContainerController(unittest.TestCase):
|
||||||
resp = self.controller.GET(req)
|
resp = self.controller.GET(req)
|
||||||
self.assertEquals(resp.status_int, 204)
|
self.assertEquals(resp.status_int, 204)
|
||||||
self.assertEquals(resp.headers.get('x-container-meta-test'), 'Value')
|
self.assertEquals(resp.headers.get('x-container-meta-test'), 'Value')
|
||||||
|
# Set another metadata header, ensuring old one doesn't disappear
|
||||||
|
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
|
||||||
|
headers={'X-Timestamp': normalize_timestamp(1),
|
||||||
|
'X-Container-Meta-Test2': 'Value2'})
|
||||||
|
resp = self.controller.POST(req)
|
||||||
|
self.assertEquals(resp.status_int, 204)
|
||||||
|
req = Request.blank('/sda1/p/a/c')
|
||||||
|
resp = self.controller.GET(req)
|
||||||
|
self.assertEquals(resp.status_int, 204)
|
||||||
|
self.assertEquals(resp.headers.get('x-container-meta-test'), 'Value')
|
||||||
|
self.assertEquals(resp.headers.get('x-container-meta-test2'), 'Value2')
|
||||||
# Update metadata header
|
# Update metadata header
|
||||||
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
|
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
|
||||||
headers={'X-Timestamp': normalize_timestamp(3),
|
headers={'X-Timestamp': normalize_timestamp(3),
|
||||||
|
|
|
@ -110,6 +110,8 @@ def fake_http_connect(*code_iter, **kwargs):
|
||||||
kwargs['give_content_type'](args[6]['content-type'])
|
kwargs['give_content_type'](args[6]['content-type'])
|
||||||
else:
|
else:
|
||||||
kwargs['give_content_type']('')
|
kwargs['give_content_type']('')
|
||||||
|
if 'give_connect' in kwargs:
|
||||||
|
kwargs['give_connect'](*args, **ckwargs)
|
||||||
status = code_iter.next()
|
status = code_iter.next()
|
||||||
etag = etag_iter.next()
|
etag = etag_iter.next()
|
||||||
if status == -1:
|
if status == -1:
|
||||||
|
@ -1750,6 +1752,137 @@ class TestContainerController(unittest.TestCase):
|
||||||
finally:
|
finally:
|
||||||
self.app.object_chunk_size = orig_object_chunk_size
|
self.app.object_chunk_size = orig_object_chunk_size
|
||||||
|
|
||||||
|
def test_PUT_metadata(self):
|
||||||
|
self.metadata_helper('PUT')
|
||||||
|
|
||||||
|
def test_POST_metadata(self):
|
||||||
|
self.metadata_helper('POST')
|
||||||
|
|
||||||
|
def metadata_helper(self, method):
|
||||||
|
for test_header, test_value in (
|
||||||
|
('X-Container-Meta-TestHeader', 'TestValue'),
|
||||||
|
('X-Container-Meta-TestHeader', '')):
|
||||||
|
test_errors = []
|
||||||
|
def test_connect(ipaddr, port, device, partition, method, path,
|
||||||
|
headers=None, query_string=None):
|
||||||
|
if path == '/a/c':
|
||||||
|
for k, v in headers.iteritems():
|
||||||
|
if k.lower() == test_header.lower() and \
|
||||||
|
v == test_value:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
test_errors.append('%s: %s not in %s' %
|
||||||
|
(test_header, test_value, headers))
|
||||||
|
with save_globals():
|
||||||
|
controller = \
|
||||||
|
proxy_server.ContainerController(self.app, 'a', 'c')
|
||||||
|
proxy_server.http_connect = fake_http_connect(200, 201, 201,
|
||||||
|
201, give_connect=test_connect)
|
||||||
|
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
|
||||||
|
headers={test_header: test_value})
|
||||||
|
req.account = 'a'
|
||||||
|
req.container = 'c'
|
||||||
|
res = getattr(controller, method)(req)
|
||||||
|
self.assertEquals(test_errors, [])
|
||||||
|
|
||||||
|
def test_PUT_bad_metadata(self):
|
||||||
|
self.bad_metadata_helper('PUT')
|
||||||
|
|
||||||
|
def test_POST_bad_metadata(self):
|
||||||
|
self.bad_metadata_helper('POST')
|
||||||
|
|
||||||
|
def bad_metadata_helper(self, method):
|
||||||
|
with save_globals():
|
||||||
|
controller = proxy_server.ContainerController(self.app, 'a', 'c')
|
||||||
|
proxy_server.http_connect = fake_http_connect(200, 201, 201, 201)
|
||||||
|
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method})
|
||||||
|
req.account = 'a'
|
||||||
|
resp = getattr(controller, method)(req)
|
||||||
|
self.assertEquals(resp.status_int, 201)
|
||||||
|
|
||||||
|
proxy_server.http_connect = fake_http_connect(201, 201, 201)
|
||||||
|
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
|
||||||
|
headers={'X-Container-Meta-' +
|
||||||
|
('a' * MAX_META_NAME_LENGTH): 'v'})
|
||||||
|
req.account = 'a'
|
||||||
|
req.container = 'c'
|
||||||
|
resp = getattr(controller, method)(req)
|
||||||
|
self.assertEquals(resp.status_int, 201)
|
||||||
|
proxy_server.http_connect = fake_http_connect(201, 201, 201)
|
||||||
|
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
|
||||||
|
headers={'X-Container-Meta-' +
|
||||||
|
('a' * (MAX_META_NAME_LENGTH + 1)): 'v'})
|
||||||
|
req.account = 'a'
|
||||||
|
req.container = 'c'
|
||||||
|
resp = getattr(controller, method)(req)
|
||||||
|
self.assertEquals(resp.status_int, 400)
|
||||||
|
|
||||||
|
proxy_server.http_connect = fake_http_connect(201, 201, 201)
|
||||||
|
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
|
||||||
|
headers={'X-Container-Meta-Too-Long':
|
||||||
|
'a' * MAX_META_VALUE_LENGTH})
|
||||||
|
req.account = 'a'
|
||||||
|
req.container = 'c'
|
||||||
|
resp = getattr(controller, method)(req)
|
||||||
|
self.assertEquals(resp.status_int, 201)
|
||||||
|
proxy_server.http_connect = fake_http_connect(201, 201, 201)
|
||||||
|
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
|
||||||
|
headers={'X-Container-Meta-Too-Long':
|
||||||
|
'a' * (MAX_META_VALUE_LENGTH + 1)})
|
||||||
|
req.account = 'a'
|
||||||
|
req.container = 'c'
|
||||||
|
resp = getattr(controller, method)(req)
|
||||||
|
self.assertEquals(resp.status_int, 400)
|
||||||
|
|
||||||
|
proxy_server.http_connect = fake_http_connect(201, 201, 201)
|
||||||
|
headers = {}
|
||||||
|
for x in xrange(MAX_META_COUNT):
|
||||||
|
headers['X-Container-Meta-%d' % x] = 'v'
|
||||||
|
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
|
||||||
|
headers=headers)
|
||||||
|
req.account = 'a'
|
||||||
|
req.container = 'c'
|
||||||
|
resp = getattr(controller, method)(req)
|
||||||
|
self.assertEquals(resp.status_int, 201)
|
||||||
|
proxy_server.http_connect = fake_http_connect(201, 201, 201)
|
||||||
|
headers = {}
|
||||||
|
for x in xrange(MAX_META_COUNT + 1):
|
||||||
|
headers['X-Container-Meta-%d' % x] = 'v'
|
||||||
|
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
|
||||||
|
headers=headers)
|
||||||
|
req.account = 'a'
|
||||||
|
req.container = 'c'
|
||||||
|
resp = getattr(controller, method)(req)
|
||||||
|
self.assertEquals(resp.status_int, 400)
|
||||||
|
|
||||||
|
proxy_server.http_connect = fake_http_connect(201, 201, 201)
|
||||||
|
headers = {}
|
||||||
|
header_value = 'a' * MAX_META_VALUE_LENGTH
|
||||||
|
size = 0
|
||||||
|
x = 0
|
||||||
|
while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:
|
||||||
|
size += 4 + MAX_META_VALUE_LENGTH
|
||||||
|
headers['X-Container-Meta-%04d' % x] = header_value
|
||||||
|
x += 1
|
||||||
|
if MAX_META_OVERALL_SIZE - size > 1:
|
||||||
|
headers['X-Container-Meta-a'] = \
|
||||||
|
'a' * (MAX_META_OVERALL_SIZE - size - 1)
|
||||||
|
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
|
||||||
|
headers=headers)
|
||||||
|
req.account = 'a'
|
||||||
|
req.container = 'c'
|
||||||
|
resp = getattr(controller, method)(req)
|
||||||
|
self.assertEquals(resp.status_int, 201)
|
||||||
|
proxy_server.http_connect = fake_http_connect(201, 201, 201)
|
||||||
|
headers['X-Container-Meta-a'] = \
|
||||||
|
'a' * (MAX_META_OVERALL_SIZE - size)
|
||||||
|
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
|
||||||
|
headers=headers)
|
||||||
|
req.account = 'a'
|
||||||
|
req.container = 'c'
|
||||||
|
resp = getattr(controller, method)(req)
|
||||||
|
self.assertEquals(resp.status_int, 400)
|
||||||
|
|
||||||
|
|
||||||
class TestAccountController(unittest.TestCase):
|
class TestAccountController(unittest.TestCase):
|
||||||
|
|
||||||
|
@ -1865,6 +1998,115 @@ class TestAccountController(unittest.TestCase):
|
||||||
finally:
|
finally:
|
||||||
self.app.object_chunk_size = orig_object_chunk_size
|
self.app.object_chunk_size = orig_object_chunk_size
|
||||||
|
|
||||||
|
def test_POST_metadata(self):
|
||||||
|
for test_header, test_value in (
|
||||||
|
('X-Account-Meta-TestHeader', 'TestValue'),
|
||||||
|
('X-Account-Meta-TestHeader', '')):
|
||||||
|
test_errors = []
|
||||||
|
def test_connect(ipaddr, port, device, partition, method, path,
|
||||||
|
headers=None, query_string=None):
|
||||||
|
for k, v in headers.iteritems():
|
||||||
|
if k.lower() == test_header.lower() and \
|
||||||
|
v == test_value:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
test_errors.append('%s: %s not in %s' %
|
||||||
|
(test_header, test_value, headers))
|
||||||
|
with save_globals():
|
||||||
|
controller = \
|
||||||
|
proxy_server.AccountController(self.app, 'a')
|
||||||
|
proxy_server.http_connect = fake_http_connect(201, 201, 201,
|
||||||
|
give_connect=test_connect)
|
||||||
|
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
|
||||||
|
headers={test_header: test_value})
|
||||||
|
req.account = 'a'
|
||||||
|
res = controller.POST(req)
|
||||||
|
self.assertEquals(test_errors, [])
|
||||||
|
|
||||||
|
def test_POST_bad_metadata(self):
|
||||||
|
with save_globals():
|
||||||
|
controller = proxy_server.AccountController(self.app, 'a')
|
||||||
|
proxy_server.http_connect = fake_http_connect(204, 204, 204)
|
||||||
|
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'})
|
||||||
|
req.account = 'a'
|
||||||
|
resp = controller.POST(req)
|
||||||
|
self.assertEquals(resp.status_int, 204)
|
||||||
|
|
||||||
|
proxy_server.http_connect = fake_http_connect(204, 204, 204)
|
||||||
|
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
|
||||||
|
headers={'X-Account-Meta-' +
|
||||||
|
('a' * MAX_META_NAME_LENGTH): 'v'})
|
||||||
|
req.account = 'a'
|
||||||
|
resp = controller.POST(req)
|
||||||
|
self.assertEquals(resp.status_int, 204)
|
||||||
|
proxy_server.http_connect = fake_http_connect(204, 204, 204)
|
||||||
|
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
|
||||||
|
headers={'X-Account-Meta-' +
|
||||||
|
('a' * (MAX_META_NAME_LENGTH + 1)): 'v'})
|
||||||
|
req.account = 'a'
|
||||||
|
resp = controller.POST(req)
|
||||||
|
self.assertEquals(resp.status_int, 400)
|
||||||
|
|
||||||
|
proxy_server.http_connect = fake_http_connect(204, 204, 204)
|
||||||
|
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
|
||||||
|
headers={'X-Account-Meta-Too-Long':
|
||||||
|
'a' * MAX_META_VALUE_LENGTH})
|
||||||
|
req.account = 'a'
|
||||||
|
resp = controller.POST(req)
|
||||||
|
self.assertEquals(resp.status_int, 204)
|
||||||
|
proxy_server.http_connect = fake_http_connect(204, 204, 204)
|
||||||
|
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
|
||||||
|
headers={'X-Account-Meta-Too-Long':
|
||||||
|
'a' * (MAX_META_VALUE_LENGTH + 1)})
|
||||||
|
req.account = 'a'
|
||||||
|
resp = controller.POST(req)
|
||||||
|
self.assertEquals(resp.status_int, 400)
|
||||||
|
|
||||||
|
proxy_server.http_connect = fake_http_connect(204, 204, 204)
|
||||||
|
headers = {}
|
||||||
|
for x in xrange(MAX_META_COUNT):
|
||||||
|
headers['X-Account-Meta-%d' % x] = 'v'
|
||||||
|
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
|
||||||
|
headers=headers)
|
||||||
|
req.account = 'a'
|
||||||
|
resp = controller.POST(req)
|
||||||
|
self.assertEquals(resp.status_int, 204)
|
||||||
|
proxy_server.http_connect = fake_http_connect(204, 204, 204)
|
||||||
|
headers = {}
|
||||||
|
for x in xrange(MAX_META_COUNT + 1):
|
||||||
|
headers['X-Account-Meta-%d' % x] = 'v'
|
||||||
|
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
|
||||||
|
headers=headers)
|
||||||
|
req.account = 'a'
|
||||||
|
resp = controller.POST(req)
|
||||||
|
self.assertEquals(resp.status_int, 400)
|
||||||
|
|
||||||
|
proxy_server.http_connect = fake_http_connect(204, 204, 204)
|
||||||
|
headers = {}
|
||||||
|
header_value = 'a' * MAX_META_VALUE_LENGTH
|
||||||
|
size = 0
|
||||||
|
x = 0
|
||||||
|
while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:
|
||||||
|
size += 4 + MAX_META_VALUE_LENGTH
|
||||||
|
headers['X-Account-Meta-%04d' % x] = header_value
|
||||||
|
x += 1
|
||||||
|
if MAX_META_OVERALL_SIZE - size > 1:
|
||||||
|
headers['X-Account-Meta-a'] = \
|
||||||
|
'a' * (MAX_META_OVERALL_SIZE - size - 1)
|
||||||
|
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
|
||||||
|
headers=headers)
|
||||||
|
req.account = 'a'
|
||||||
|
resp = controller.POST(req)
|
||||||
|
self.assertEquals(resp.status_int, 204)
|
||||||
|
proxy_server.http_connect = fake_http_connect(204, 204, 204)
|
||||||
|
headers['X-Account-Meta-a'] = \
|
||||||
|
'a' * (MAX_META_OVERALL_SIZE - size)
|
||||||
|
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
|
||||||
|
headers=headers)
|
||||||
|
req.account = 'a'
|
||||||
|
resp = controller.POST(req)
|
||||||
|
self.assertEquals(resp.status_int, 400)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
Loading…
Reference in New Issue