Merge "container-server: return objects of a given policy"
This commit is contained in:
commit
71f8e50b0e
|
@ -768,16 +768,22 @@ class ContainerController(BaseStorageServer):
|
|||
include_deleted=include_deleted, fill_gaps=fill_gaps,
|
||||
include_own=include_own)
|
||||
else:
|
||||
requested_policy_index = self.get_and_validate_policy_index(req)
|
||||
resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
|
||||
if is_deleted:
|
||||
return HTTPNotFound(request=req, headers=resp_headers)
|
||||
resp_headers['X-Backend-Record-Type'] = 'object'
|
||||
storage_policy_index = (
|
||||
requested_policy_index if requested_policy_index is not None
|
||||
else info['storage_policy_index'])
|
||||
resp_headers['X-Backend-Record-Storage-Policy-Index'] = \
|
||||
storage_policy_index
|
||||
# Use the retired db while container is in process of sharding,
|
||||
# otherwise use current db
|
||||
src_broker = broker.get_brokers()[0]
|
||||
container_list = src_broker.list_objects_iter(
|
||||
limit, marker, end_marker, prefix, delimiter, path,
|
||||
storage_policy_index=info['storage_policy_index'],
|
||||
storage_policy_index=storage_policy_index,
|
||||
reverse=reverse, allow_reserved=req.allow_reserved_names)
|
||||
return self.create_listing(req, out_content_type, info, resp_headers,
|
||||
broker.metadata, container_list, container)
|
||||
|
|
|
@ -2285,7 +2285,9 @@ class Controller(object):
|
|||
:param req: original Request instance.
|
||||
:param account: account in which `container` is stored.
|
||||
:param container: container from which listing should be fetched.
|
||||
:param headers: headers to be included with the request
|
||||
:param headers: extra headers to be included with the listing
|
||||
sub-request; these update the headers copied from the original
|
||||
request.
|
||||
:param params: query string parameters to be used.
|
||||
:return: a tuple of (deserialized json data structure, swob Response)
|
||||
"""
|
||||
|
|
|
@ -318,6 +318,17 @@ class ContainerController(Controller):
|
|||
# shard may return the root's shard range.
|
||||
shard_listing_history = req.environ.setdefault(
|
||||
'swift.shard_listing_history', [])
|
||||
policy_key = 'X-Backend-Storage-Policy-Index'
|
||||
if not (shard_listing_history or policy_key in req.headers):
|
||||
# We're handling the original request to the root container: set
|
||||
# the root policy index in the request, unless it is already set,
|
||||
# so that shards will return listings for that policy index.
|
||||
# Note: we only get here if the root responded with shard ranges,
|
||||
# or if the shard ranges were cached and the cached root container
|
||||
# info has sharding_state==sharded; in both cases we can assume
|
||||
# that the response is "modern enough" to include
|
||||
# 'X-Backend-Storage-Policy-Index'.
|
||||
req.headers[policy_key] = resp.headers[policy_key]
|
||||
shard_listing_history.append((self.account_name, self.container_name))
|
||||
shard_ranges = [ShardRange.from_dict(data)
|
||||
for data in json.loads(resp.body)]
|
||||
|
|
|
@ -16,6 +16,7 @@ import json
|
|||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from nose import SkipTest
|
||||
|
@ -23,6 +24,7 @@ import six
|
|||
from six.moves.urllib.parse import quote
|
||||
|
||||
from swift.common import direct_client, utils
|
||||
from swift.common.internal_client import UnexpectedResponse
|
||||
from swift.common.manager import Manager
|
||||
from swift.common.memcached import MemcacheRing
|
||||
from swift.common.utils import ShardRange, parse_db_filename, get_db_files, \
|
||||
|
@ -39,7 +41,7 @@ from test import annotate_failure
|
|||
from test.probe import PROXY_BASE_URL
|
||||
from test.probe.brain import BrainSplitter
|
||||
from test.probe.common import ReplProbeTest, get_server_number, \
|
||||
wait_for_server_to_hangup
|
||||
wait_for_server_to_hangup, ENABLED_POLICIES
|
||||
import mock
|
||||
|
||||
|
||||
|
@ -2215,6 +2217,72 @@ class TestContainerSharding(BaseTestContainerSharding):
|
|||
self.assert_container_delete_fails()
|
||||
self.assert_container_post_ok('revived')
|
||||
|
||||
def _do_test_sharded_can_get_objects_different_policy(self,
|
||||
policy_idx,
|
||||
new_policy_idx):
|
||||
# create sharded container
|
||||
client.delete_container(self.url, self.token, self.container_name)
|
||||
self.brain.put_container(policy_index=int(policy_idx))
|
||||
all_obj_names = self._make_object_names(self.max_shard_size)
|
||||
self.put_objects(all_obj_names)
|
||||
client.post_container(self.url, self.admin_token, self.container_name,
|
||||
headers={'X-Container-Sharding': 'on'})
|
||||
for n in self.brain.node_numbers:
|
||||
self.sharders.once(
|
||||
number=n, additional_args='--partitions=%s' % self.brain.part)
|
||||
# empty and delete
|
||||
self.delete_objects(all_obj_names)
|
||||
shard_ranges = self.get_container_shard_ranges()
|
||||
self.run_sharders(shard_ranges)
|
||||
client.delete_container(self.url, self.token, self.container_name)
|
||||
|
||||
# re-create with new_policy_idx
|
||||
self.brain.put_container(policy_index=int(new_policy_idx))
|
||||
|
||||
# we re-use shard ranges
|
||||
new_shard_ranges = self.get_container_shard_ranges()
|
||||
self.assertEqual(shard_ranges, new_shard_ranges)
|
||||
self.put_objects(all_obj_names)
|
||||
|
||||
# The shard is still on the old policy index, but the root spi
|
||||
# is passed to shard container server and is used to pull objects
|
||||
# of that index out.
|
||||
self.assert_container_listing(all_obj_names)
|
||||
# although a head request is getting object count for the shard spi
|
||||
self.assert_container_object_count(0)
|
||||
|
||||
# we can force the listing to use the old policy index in which case we
|
||||
# expect no objects to be listed
|
||||
try:
|
||||
resp = self.internal_client.make_request(
|
||||
'GET',
|
||||
path=self.internal_client.make_path(
|
||||
self.account, self.container_name),
|
||||
headers={'X-Backend-Storage-Policy-Index': str(policy_idx)},
|
||||
acceptable_statuses=(2,),
|
||||
params={'format': 'json'}
|
||||
)
|
||||
except UnexpectedResponse as exc:
|
||||
self.fail('Listing failed with %s' % exc.resp.status)
|
||||
|
||||
self.assertEqual([], json.loads(b''.join(resp.app_iter)))
|
||||
|
||||
@unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy")
|
||||
def test_sharded_can_get_objects_different_policy(self):
|
||||
policy_idx = self.policy.idx
|
||||
new_policy_idx = [pol.idx for pol in ENABLED_POLICIES
|
||||
if pol != self.policy.idx][0]
|
||||
self._do_test_sharded_can_get_objects_different_policy(
|
||||
policy_idx, new_policy_idx)
|
||||
|
||||
@unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy")
|
||||
def test_sharded_can_get_objects_different_policy_reversed(self):
|
||||
policy_idx = [pol.idx for pol in ENABLED_POLICIES
|
||||
if pol != self.policy][0]
|
||||
new_policy_idx = self.policy.idx
|
||||
self._do_test_sharded_can_get_objects_different_policy(
|
||||
policy_idx, new_policy_idx)
|
||||
|
||||
def test_object_update_redirection(self):
|
||||
all_obj_names = self._make_object_names(self.max_shard_size)
|
||||
self.put_objects(all_obj_names)
|
||||
|
|
|
@ -3415,6 +3415,12 @@ class TestContainerController(unittest.TestCase):
|
|||
self.assertIn('X-Backend-Record-Type', resp.headers)
|
||||
self.assertEqual(
|
||||
'object', resp.headers.pop('X-Backend-Record-Type'))
|
||||
self.assertEqual(
|
||||
str(POLICIES.default.idx),
|
||||
resp.headers.pop('X-Backend-Storage-Policy-Index'))
|
||||
self.assertEqual(
|
||||
str(POLICIES.default.idx),
|
||||
resp.headers.pop('X-Backend-Record-Storage-Policy-Index'))
|
||||
resp.headers.pop('Content-Length')
|
||||
return resp
|
||||
|
||||
|
@ -3430,6 +3436,11 @@ class TestContainerController(unittest.TestCase):
|
|||
self.assertIn('X-Backend-Record-Type', resp.headers)
|
||||
self.assertEqual(
|
||||
'shard', resp.headers.pop('X-Backend-Record-Type'))
|
||||
self.assertEqual(
|
||||
str(POLICIES.default.idx),
|
||||
resp.headers.pop('X-Backend-Storage-Policy-Index'))
|
||||
self.assertNotIn('X-Backend-Record-Storage-Policy-Index',
|
||||
resp.headers)
|
||||
resp.headers.pop('Content-Length')
|
||||
return resp
|
||||
|
||||
|
@ -4239,6 +4250,119 @@ class TestContainerController(unittest.TestCase):
|
|||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 406)
|
||||
|
||||
@patch_policies([
|
||||
StoragePolicy(0, name='nulo', is_default=True),
|
||||
StoragePolicy(1, name='unu'),
|
||||
StoragePolicy(2, name='du'),
|
||||
])
|
||||
def test_GET_objects_of_different_policies(self):
|
||||
# make a container
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
|
||||
'HTTP_X_TIMESTAMP': '0'})
|
||||
resp = req.get_response(self.controller)
|
||||
resp_policy_idx = resp.headers['X-Backend-Storage-Policy-Index']
|
||||
self.assertEqual(resp_policy_idx, str(POLICIES.default.idx))
|
||||
|
||||
pol_def_objs = ['obj_default_%d' % i for i in range(11)]
|
||||
pol_1_objs = ['obj_1_%d' % i for i in range(10)]
|
||||
|
||||
# fill the container
|
||||
for obj in pol_def_objs:
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/%s' % obj,
|
||||
environ={
|
||||
'REQUEST_METHOD': 'PUT',
|
||||
'HTTP_X_TIMESTAMP': '1',
|
||||
'HTTP_X_CONTENT_TYPE': 'text/plain',
|
||||
'HTTP_X_ETAG': 'x',
|
||||
'HTTP_X_SIZE': 0})
|
||||
self._update_object_put_headers(req)
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
|
||||
for obj in pol_1_objs:
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c/%s' % obj,
|
||||
environ={
|
||||
'REQUEST_METHOD': 'PUT',
|
||||
'HTTP_X_TIMESTAMP': '1',
|
||||
'HTTP_X_CONTENT_TYPE': 'text/plain',
|
||||
'HTTP_X_ETAG': 'x',
|
||||
'HTTP_X_SIZE': 0,
|
||||
'HTTP_X_BACKEND_STORAGE_POLICY_INDEX': 1})
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 201)
|
||||
|
||||
expected_pol_def_objs = [o.encode('utf8') for o in pol_def_objs]
|
||||
expected_pol_1_objs = [o.encode('utf8') for o in pol_1_objs]
|
||||
|
||||
# By default the container server will return objects belonging to
|
||||
# the brokers storage policy
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 200)
|
||||
result = [o for o in resp.body.split(b'\n') if o]
|
||||
self.assertEqual(len(result), 11)
|
||||
self.assertEqual(sorted(result), sorted(expected_pol_def_objs))
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
|
||||
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
|
||||
self.assertEqual('0',
|
||||
resp.headers['X-Backend-Record-Storage-Policy-Index'])
|
||||
|
||||
# If we specify the policy 0 idx we should get the same
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
req.headers['X-Backend-Storage-Policy-Index'] = POLICIES.default.idx
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 200)
|
||||
result = [o for o in resp.body.split(b'\n') if o]
|
||||
self.assertEqual(len(result), 11)
|
||||
self.assertEqual(sorted(result), sorted(expected_pol_def_objs))
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
|
||||
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
|
||||
self.assertEqual('0',
|
||||
resp.headers['X-Backend-Record-Storage-Policy-Index'])
|
||||
|
||||
# And if we specify a different idx we'll get objects for that policy
|
||||
# and the X-Backend-Record-Storage-Policy-Index letting us know the
|
||||
# policy for which these objects came from, if it differs from the
|
||||
# policy stored in the DB.
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
req.headers['X-Backend-Storage-Policy-Index'] = 1
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 200)
|
||||
result = [o for o in resp.body.split(b'\n') if o]
|
||||
self.assertEqual(len(result), 10)
|
||||
self.assertEqual(sorted(result), sorted(expected_pol_1_objs))
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
|
||||
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
|
||||
self.assertEqual('1',
|
||||
resp.headers['X-Backend-Record-Storage-Policy-Index'])
|
||||
|
||||
# And an index that the broker doesn't have any objects for
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
req.headers['X-Backend-Storage-Policy-Index'] = 2
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 204)
|
||||
result = [o for o in resp.body.split(b'\n') if o]
|
||||
self.assertEqual(len(result), 0)
|
||||
self.assertFalse(result)
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
|
||||
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
|
||||
self.assertEqual('2',
|
||||
resp.headers['X-Backend-Record-Storage-Policy-Index'])
|
||||
|
||||
# And an index that doesn't exist in POLICIES
|
||||
req = Request.blank(
|
||||
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
req.headers['X-Backend-Storage-Policy-Index'] = 3
|
||||
resp = req.get_response(self.controller)
|
||||
self.assertEqual(resp.status_int, 400)
|
||||
|
||||
def test_GET_limit(self):
|
||||
# make a container
|
||||
req = Request.blank(
|
||||
|
|
|
@ -610,25 +610,30 @@ class TestContainerController(TestRingBase):
|
|||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='', end_marker='ham\x00', limit=str(limit),
|
||||
states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='\xd1\xb0', end_marker='\xf0\x9f\x8c\xb4\x00',
|
||||
states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1]
|
||||
+ sr_objs[2])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[4].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='\xe2\xa8\x83', end_marker='', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2]
|
||||
+ sr_objs[3])))), # 200
|
||||
|
@ -653,13 +658,19 @@ class TestContainerController(TestRingBase):
|
|||
# path, headers, params
|
||||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(states='listing')), # 200
|
||||
(shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(shard_ranges[0].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='', end_marker='ham\x00', limit=str(limit),
|
||||
states='listing')), # 200
|
||||
(shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))), # 200
|
||||
(root_range.name, {'X-Backend-Record-Type': 'object'},
|
||||
(root_range.name,
|
||||
{'X-Backend-Record-Type': 'object',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='p', end_marker='',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200
|
||||
]
|
||||
|
@ -685,27 +696,32 @@ class TestContainerController(TestRingBase):
|
|||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(states='listing', reverse='true', limit='')),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[4].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='', end_marker='\xf0\x9f\x8c\xb4', states='listing',
|
||||
reverse='true', limit=str(limit))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='\xf0\x9f\x8c\xb5', end_marker='\xe2\x98\x83',
|
||||
states='listing', reverse='true',
|
||||
limit=str(limit - len(sr_objs[4])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='\xe2\x98\x84', end_marker='pie', states='listing',
|
||||
reverse='true',
|
||||
limit=str(limit - len(sr_objs[4] + sr_objs[3])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='q', end_marker='ham', states='listing',
|
||||
reverse='true',
|
||||
limit=str(limit - len(sr_objs[4] + sr_objs[3]
|
||||
+ sr_objs[2])))), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
|
||||
{'X-Backend-Record-Type': 'auto'},
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='i', end_marker='', states='listing', reverse='true',
|
||||
limit=str(limit - len(sr_objs[4] + sr_objs[3] + sr_objs[2]
|
||||
+ sr_objs[1])))), # 200
|
||||
|
@ -735,15 +751,18 @@ class TestContainerController(TestRingBase):
|
|||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(limit=str(limit), states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='', end_marker='ham\x00', states='listing',
|
||||
limit=str(limit))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))),
|
||||
]
|
||||
|
@ -771,15 +790,18 @@ class TestContainerController(TestRingBase):
|
|||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(marker=marker, states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00',
|
||||
states='listing', limit=str(limit))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00',
|
||||
states='listing', limit=str(limit))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[4].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='\xe2\xa8\x83', end_marker='', states='listing',
|
||||
limit=str(limit - len(sr_objs[3][2:])))),
|
||||
]
|
||||
|
@ -809,28 +831,34 @@ class TestContainerController(TestRingBase):
|
|||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(end_marker=end_marker, states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='', end_marker='ham\x00', states='listing',
|
||||
limit=str(limit))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 404
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 404
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 404
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 404
|
||||
dict(marker='\xd1\xb0', end_marker=end_marker, states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1]
|
||||
+ sr_objs[2])))),
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker='\xd1\xb0', end_marker=end_marker, states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1]
|
||||
+ sr_objs[2])))),
|
||||
|
@ -856,7 +884,8 @@ class TestContainerController(TestRingBase):
|
|||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(prefix=prefix, states='listing')), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 404
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 404
|
||||
dict(prefix=prefix, marker='', end_marker='pie\x00',
|
||||
states='listing', limit=str(limit))),
|
||||
]
|
||||
|
@ -877,7 +906,8 @@ class TestContainerController(TestRingBase):
|
|||
dict(states='listing', limit=str(limit),
|
||||
marker=marker, end_marker=end_marker)), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker=marker, end_marker=end_marker, states='listing',
|
||||
limit=str(limit))),
|
||||
]
|
||||
|
@ -898,7 +928,8 @@ class TestContainerController(TestRingBase):
|
|||
dict(marker=end_marker, reverse='true', end_marker=marker,
|
||||
limit=str(limit), states='listing',)), # 200
|
||||
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
|
||||
{'X-Backend-Record-Type': 'auto'}, # 200
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'}, # 200
|
||||
dict(marker=end_marker, end_marker=marker, states='listing',
|
||||
limit=str(limit), reverse='true')),
|
||||
]
|
||||
|
@ -1720,28 +1751,160 @@ class TestContainerController(TestRingBase):
|
|||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(states='listing')), # 200
|
||||
# get first shard objects
|
||||
(shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(shard_ranges[0].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='', end_marker='ham\x00', states='listing',
|
||||
limit=str(limit))), # 200
|
||||
# get second shard sub-shard ranges
|
||||
(shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
# get first sub-shard objects
|
||||
(sub_shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(sub_shard_ranges[0].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='juice\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
# get second sub-shard objects
|
||||
(sub_shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(sub_shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='j', end_marker='lemon\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))),
|
||||
# get remainder of first shard objects
|
||||
(shard_ranges[1].name, {'X-Backend-Record-Type': 'object'},
|
||||
(shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'object',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='l', end_marker='pie\x00',
|
||||
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] +
|
||||
sub_sr_objs[1])))), # 200
|
||||
# get third shard objects
|
||||
(shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'},
|
||||
(shard_ranges[2].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='p', end_marker='', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200
|
||||
]
|
||||
expected_objects = (
|
||||
sr_objs[0] + sub_sr_objs[0] + sub_sr_objs[1] +
|
||||
sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):] + sr_objs[2])
|
||||
resp = self._check_GET_shard_listing(
|
||||
mock_responses, expected_objects, expected_requests)
|
||||
# root object count will overridden by actual length of listing
|
||||
self.check_response(resp, root_resp_hdrs)
|
||||
|
||||
@patch_policies([
|
||||
StoragePolicy(0, 'zero', True, object_ring=FakeRing()),
|
||||
StoragePolicy(1, 'one', False, object_ring=FakeRing())
|
||||
])
|
||||
def test_GET_sharded_container_sharding_shard_mixed_policies(self):
|
||||
# scenario: one shard is in process of sharding, shards have different
|
||||
# policy than root, expect listing to always request root policy index
|
||||
shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', ''))
|
||||
shard_ranges = [
|
||||
ShardRange('.shards_a/c_' + upper, Timestamp.now(), lower, upper)
|
||||
for lower, upper in shard_bounds]
|
||||
sr_dicts = [dict(sr) for sr in shard_ranges]
|
||||
sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges]
|
||||
shard_resp_hdrs = [
|
||||
{'X-Backend-Sharding-State': 'unsharded',
|
||||
'X-Container-Object-Count': len(sr_objs[i]),
|
||||
'X-Container-Bytes-Used':
|
||||
sum([obj['bytes'] for obj in sr_objs[i]]),
|
||||
'X-Container-Meta-Flavour': 'flavour%d' % i,
|
||||
'X-Backend-Storage-Policy-Index': 1,
|
||||
'X-Backend-Record-Storage-Policy-Index': 0}
|
||||
for i in range(3)]
|
||||
shard_1_shard_resp_hdrs = dict(shard_resp_hdrs[1])
|
||||
shard_1_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard'
|
||||
|
||||
# second shard is sharding and has cleaved two out of three sub shards
|
||||
shard_resp_hdrs[1]['X-Backend-Sharding-State'] = 'sharding'
|
||||
sub_shard_bounds = (('ham', 'juice'), ('juice', 'lemon'))
|
||||
sub_shard_ranges = [
|
||||
ShardRange('a/c_sub_' + upper, Timestamp.now(), lower, upper)
|
||||
for lower, upper in sub_shard_bounds]
|
||||
sub_sr_dicts = [dict(sr) for sr in sub_shard_ranges]
|
||||
sub_sr_objs = [self._make_shard_objects(sr) for sr in sub_shard_ranges]
|
||||
sub_shard_resp_hdrs = [
|
||||
{'X-Backend-Sharding-State': 'unsharded',
|
||||
'X-Container-Object-Count': len(sub_sr_objs[i]),
|
||||
'X-Container-Bytes-Used':
|
||||
sum([obj['bytes'] for obj in sub_sr_objs[i]]),
|
||||
'X-Container-Meta-Flavour': 'flavour%d' % i,
|
||||
'X-Backend-Storage-Policy-Index': 1,
|
||||
'X-Backend-Record-Storage-Policy-Index': 0}
|
||||
for i in range(2)]
|
||||
|
||||
all_objects = []
|
||||
for objects in sr_objs:
|
||||
all_objects.extend(objects)
|
||||
size_all_objects = sum([obj['bytes'] for obj in all_objects])
|
||||
num_all_objects = len(all_objects)
|
||||
limit = CONTAINER_LISTING_LIMIT
|
||||
root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded',
|
||||
'X-Backend-Timestamp': '99',
|
||||
'X-Container-Object-Count': num_all_objects,
|
||||
'X-Container-Bytes-Used': size_all_objects,
|
||||
'X-Container-Meta-Flavour': 'peach',
|
||||
'X-Backend-Storage-Policy-Index': 0}
|
||||
root_shard_resp_hdrs = dict(root_resp_hdrs)
|
||||
root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard'
|
||||
|
||||
mock_responses = [
|
||||
# status, body, headers
|
||||
(200, sr_dicts, root_shard_resp_hdrs),
|
||||
(200, sr_objs[0], shard_resp_hdrs[0]),
|
||||
(200, sub_sr_dicts + [sr_dicts[1]], shard_1_shard_resp_hdrs),
|
||||
(200, sub_sr_objs[0], sub_shard_resp_hdrs[0]),
|
||||
(200, sub_sr_objs[1], sub_shard_resp_hdrs[1]),
|
||||
(200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):],
|
||||
shard_resp_hdrs[1]),
|
||||
(200, sr_objs[2], shard_resp_hdrs[2])
|
||||
]
|
||||
# NB marker always advances to last object name
|
||||
expected_requests = [
|
||||
# get root shard ranges
|
||||
('a/c', {'X-Backend-Record-Type': 'auto'},
|
||||
dict(states='listing')), # 200
|
||||
# get first shard objects
|
||||
(shard_ranges[0].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='', end_marker='ham\x00', states='listing',
|
||||
limit=str(limit))), # 200
|
||||
# get second shard sub-shard ranges
|
||||
(shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='pie\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
# get first sub-shard objects
|
||||
(sub_shard_ranges[0].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='h', end_marker='juice\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0])))),
|
||||
# get second sub-shard objects
|
||||
(sub_shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='j', end_marker='lemon\x00', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))),
|
||||
# get remainder of second shard objects
|
||||
(shard_ranges[1].name,
|
||||
{'X-Backend-Record-Type': 'object',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='l', end_marker='pie\x00',
|
||||
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] +
|
||||
sub_sr_objs[1])))), # 200
|
||||
# get third shard objects
|
||||
(shard_ranges[2].name,
|
||||
{'X-Backend-Record-Type': 'auto',
|
||||
'X-Backend-Storage-Policy-Index': '0'},
|
||||
dict(marker='p', end_marker='', states='listing',
|
||||
limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200
|
||||
]
|
||||
|
@ -1927,6 +2090,88 @@ class TestContainerController(TestRingBase):
|
|||
('delete', 'shard-listing/a/c', None, None)],
|
||||
self.memcache.calls)
|
||||
|
||||
def test_get_from_shards_add_root_spi(self):
|
||||
self._setup_shard_range_stubs()
|
||||
shard_resp = mock.MagicMock(status_int=204, headers={})
|
||||
|
||||
def mock_get_container_listing(self_, req, *args, **kargs):
|
||||
captured_hdrs.update(req.headers)
|
||||
return None, shard_resp
|
||||
|
||||
# header in response -> header added to request
|
||||
captured_hdrs = {}
|
||||
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
resp = mock.MagicMock(body=self._stub_shards_dump,
|
||||
headers=self.root_resp_hdrs,
|
||||
request=req)
|
||||
resp.headers['X-Backend-Storage-Policy-Index'] = '0'
|
||||
with mock.patch('swift.proxy.controllers.container.'
|
||||
'ContainerController._get_container_listing',
|
||||
mock_get_container_listing):
|
||||
controller_cls, d = self.app.get_controller(req)
|
||||
controller = controller_cls(self.app, **d)
|
||||
controller._get_from_shards(req, resp)
|
||||
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs)
|
||||
self.assertEqual(
|
||||
captured_hdrs['X-Backend-Storage-Policy-Index'], '0')
|
||||
|
||||
captured_hdrs = {}
|
||||
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
resp = mock.MagicMock(body=self._stub_shards_dump,
|
||||
headers=self.root_resp_hdrs,
|
||||
request=req)
|
||||
resp.headers['X-Backend-Storage-Policy-Index'] = '1'
|
||||
with mock.patch('swift.proxy.controllers.container.'
|
||||
'ContainerController._get_container_listing',
|
||||
mock_get_container_listing):
|
||||
controller_cls, d = self.app.get_controller(req)
|
||||
controller = controller_cls(self.app, **d)
|
||||
controller._get_from_shards(req, resp)
|
||||
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs)
|
||||
self.assertEqual(
|
||||
captured_hdrs['X-Backend-Storage-Policy-Index'], '1')
|
||||
|
||||
# header not added to request if not root request
|
||||
captured_hdrs = {}
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={
|
||||
'REQUEST_METHOD': 'GET',
|
||||
'swift.shard_listing_history': [('a', 'c')]}
|
||||
)
|
||||
resp = mock.MagicMock(body=self._stub_shards_dump,
|
||||
headers=self.root_resp_hdrs,
|
||||
request=req)
|
||||
resp.headers['X-Backend-Storage-Policy-Index'] = '0'
|
||||
with mock.patch('swift.proxy.controllers.container.'
|
||||
'ContainerController._get_container_listing',
|
||||
mock_get_container_listing):
|
||||
controller_cls, d = self.app.get_controller(req)
|
||||
controller = controller_cls(self.app, **d)
|
||||
controller._get_from_shards(req, resp)
|
||||
|
||||
self.assertNotIn('X-Backend-Storage-Policy-Index', captured_hdrs)
|
||||
|
||||
# existing X-Backend-Storage-Policy-Index in request is respected
|
||||
captured_hdrs = {}
|
||||
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
|
||||
req.headers['X-Backend-Storage-Policy-Index'] = '0'
|
||||
resp = mock.MagicMock(body=self._stub_shards_dump,
|
||||
headers=self.root_resp_hdrs,
|
||||
request=req)
|
||||
resp.headers['X-Backend-Storage-Policy-Index'] = '1'
|
||||
with mock.patch('swift.proxy.controllers.container.'
|
||||
'ContainerController._get_container_listing',
|
||||
mock_get_container_listing):
|
||||
controller_cls, d = self.app.get_controller(req)
|
||||
controller = controller_cls(self.app, **d)
|
||||
controller._get_from_shards(req, resp)
|
||||
|
||||
self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs)
|
||||
self.assertEqual(
|
||||
captured_hdrs['X-Backend-Storage-Policy-Index'], '0')
|
||||
|
||||
def test_GET_shard_ranges(self):
|
||||
self._setup_shard_range_stubs()
|
||||
# expect shard ranges cache time to be default value of 600
|
||||
|
|
Loading…
Reference in New Issue