Merge "container-server: return objects of a given policy"

This commit is contained in:
Zuul 2021-08-16 23:01:05 +00:00 committed by Gerrit Code Review
commit 71f8e50b0e
6 changed files with 493 additions and 37 deletions

View File

@ -768,16 +768,22 @@ class ContainerController(BaseStorageServer):
include_deleted=include_deleted, fill_gaps=fill_gaps,
include_own=include_own)
else:
requested_policy_index = self.get_and_validate_policy_index(req)
resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
if is_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
resp_headers['X-Backend-Record-Type'] = 'object'
storage_policy_index = (
requested_policy_index if requested_policy_index is not None
else info['storage_policy_index'])
resp_headers['X-Backend-Record-Storage-Policy-Index'] = \
storage_policy_index
# Use the retired db while container is in process of sharding,
# otherwise use current db
src_broker = broker.get_brokers()[0]
container_list = src_broker.list_objects_iter(
limit, marker, end_marker, prefix, delimiter, path,
storage_policy_index=info['storage_policy_index'],
storage_policy_index=storage_policy_index,
reverse=reverse, allow_reserved=req.allow_reserved_names)
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)

View File

@ -2285,7 +2285,9 @@ class Controller(object):
:param req: original Request instance.
:param account: account in which `container` is stored.
:param container: container from which listing should be fetched.
:param headers: headers to be included with the request
:param headers: extra headers to be included with the listing
sub-request; these update the headers copied from the original
request.
:param params: query string parameters to be used.
:return: a tuple of (deserialized json data structure, swob Response)
"""

View File

@ -318,6 +318,17 @@ class ContainerController(Controller):
# shard may return the root's shard range.
shard_listing_history = req.environ.setdefault(
'swift.shard_listing_history', [])
policy_key = 'X-Backend-Storage-Policy-Index'
if not (shard_listing_history or policy_key in req.headers):
# We're handling the original request to the root container: set
# the root policy index in the request, unless it is already set,
# so that shards will return listings for that policy index.
# Note: we only get here if the root responded with shard ranges,
# or if the shard ranges were cached and the cached root container
# info has sharding_state==sharded; in both cases we can assume
# that the response is "modern enough" to include
# 'X-Backend-Storage-Policy-Index'.
req.headers[policy_key] = resp.headers[policy_key]
shard_listing_history.append((self.account_name, self.container_name))
shard_ranges = [ShardRange.from_dict(data)
for data in json.loads(resp.body)]

View File

@ -16,6 +16,7 @@ import json
import os
import shutil
import subprocess
import unittest
import uuid
from nose import SkipTest
@ -23,6 +24,7 @@ import six
from six.moves.urllib.parse import quote
from swift.common import direct_client, utils
from swift.common.internal_client import UnexpectedResponse
from swift.common.manager import Manager
from swift.common.memcached import MemcacheRing
from swift.common.utils import ShardRange, parse_db_filename, get_db_files, \
@ -39,7 +41,7 @@ from test import annotate_failure
from test.probe import PROXY_BASE_URL
from test.probe.brain import BrainSplitter
from test.probe.common import ReplProbeTest, get_server_number, \
wait_for_server_to_hangup
wait_for_server_to_hangup, ENABLED_POLICIES
import mock
@ -2215,6 +2217,72 @@ class TestContainerSharding(BaseTestContainerSharding):
self.assert_container_delete_fails()
self.assert_container_post_ok('revived')
def _do_test_sharded_can_get_objects_different_policy(self,
policy_idx,
new_policy_idx):
# create sharded container
client.delete_container(self.url, self.token, self.container_name)
self.brain.put_container(policy_index=int(policy_idx))
all_obj_names = self._make_object_names(self.max_shard_size)
self.put_objects(all_obj_names)
client.post_container(self.url, self.admin_token, self.container_name,
headers={'X-Container-Sharding': 'on'})
for n in self.brain.node_numbers:
self.sharders.once(
number=n, additional_args='--partitions=%s' % self.brain.part)
# empty and delete
self.delete_objects(all_obj_names)
shard_ranges = self.get_container_shard_ranges()
self.run_sharders(shard_ranges)
client.delete_container(self.url, self.token, self.container_name)
# re-create with new_policy_idx
self.brain.put_container(policy_index=int(new_policy_idx))
# we re-use shard ranges
new_shard_ranges = self.get_container_shard_ranges()
self.assertEqual(shard_ranges, new_shard_ranges)
self.put_objects(all_obj_names)
# The shard is still on the old policy index, but the root spi
# is passed to shard container server and is used to pull objects
# of that index out.
self.assert_container_listing(all_obj_names)
# although a head request is getting object count for the shard spi
self.assert_container_object_count(0)
# we can force the listing to use the old policy index in which case we
# expect no objects to be listed
try:
resp = self.internal_client.make_request(
'GET',
path=self.internal_client.make_path(
self.account, self.container_name),
headers={'X-Backend-Storage-Policy-Index': str(policy_idx)},
acceptable_statuses=(2,),
params={'format': 'json'}
)
except UnexpectedResponse as exc:
self.fail('Listing failed with %s' % exc.resp.status)
self.assertEqual([], json.loads(b''.join(resp.app_iter)))
@unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy")
def test_sharded_can_get_objects_different_policy(self):
policy_idx = self.policy.idx
new_policy_idx = [pol.idx for pol in ENABLED_POLICIES
if pol != self.policy.idx][0]
self._do_test_sharded_can_get_objects_different_policy(
policy_idx, new_policy_idx)
@unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy")
def test_sharded_can_get_objects_different_policy_reversed(self):
policy_idx = [pol.idx for pol in ENABLED_POLICIES
if pol != self.policy][0]
new_policy_idx = self.policy.idx
self._do_test_sharded_can_get_objects_different_policy(
policy_idx, new_policy_idx)
def test_object_update_redirection(self):
all_obj_names = self._make_object_names(self.max_shard_size)
self.put_objects(all_obj_names)

View File

@ -3415,6 +3415,12 @@ class TestContainerController(unittest.TestCase):
self.assertIn('X-Backend-Record-Type', resp.headers)
self.assertEqual(
'object', resp.headers.pop('X-Backend-Record-Type'))
self.assertEqual(
str(POLICIES.default.idx),
resp.headers.pop('X-Backend-Storage-Policy-Index'))
self.assertEqual(
str(POLICIES.default.idx),
resp.headers.pop('X-Backend-Record-Storage-Policy-Index'))
resp.headers.pop('Content-Length')
return resp
@ -3430,6 +3436,11 @@ class TestContainerController(unittest.TestCase):
self.assertIn('X-Backend-Record-Type', resp.headers)
self.assertEqual(
'shard', resp.headers.pop('X-Backend-Record-Type'))
self.assertEqual(
str(POLICIES.default.idx),
resp.headers.pop('X-Backend-Storage-Policy-Index'))
self.assertNotIn('X-Backend-Record-Storage-Policy-Index',
resp.headers)
resp.headers.pop('Content-Length')
return resp
@ -4239,6 +4250,119 @@ class TestContainerController(unittest.TestCase):
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 406)
@patch_policies([
StoragePolicy(0, name='nulo', is_default=True),
StoragePolicy(1, name='unu'),
StoragePolicy(2, name='du'),
])
def test_GET_objects_of_different_policies(self):
# make a container
req = Request.blank(
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = req.get_response(self.controller)
resp_policy_idx = resp.headers['X-Backend-Storage-Policy-Index']
self.assertEqual(resp_policy_idx, str(POLICIES.default.idx))
pol_def_objs = ['obj_default_%d' % i for i in range(11)]
pol_1_objs = ['obj_1_%d' % i for i in range(10)]
# fill the container
for obj in pol_def_objs:
req = Request.blank(
'/sda1/p/a/c/%s' % obj,
environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x',
'HTTP_X_SIZE': 0})
self._update_object_put_headers(req)
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
for obj in pol_1_objs:
req = Request.blank(
'/sda1/p/a/c/%s' % obj,
environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x',
'HTTP_X_SIZE': 0,
'HTTP_X_BACKEND_STORAGE_POLICY_INDEX': 1})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
expected_pol_def_objs = [o.encode('utf8') for o in pol_def_objs]
expected_pol_1_objs = [o.encode('utf8') for o in pol_1_objs]
# By default the container server will return objects belonging to
# the brokers storage policy
req = Request.blank(
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200)
result = [o for o in resp.body.split(b'\n') if o]
self.assertEqual(len(result), 11)
self.assertEqual(sorted(result), sorted(expected_pol_def_objs))
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
self.assertEqual('0',
resp.headers['X-Backend-Record-Storage-Policy-Index'])
# If we specify the policy 0 idx we should get the same
req = Request.blank(
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
req.headers['X-Backend-Storage-Policy-Index'] = POLICIES.default.idx
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200)
result = [o for o in resp.body.split(b'\n') if o]
self.assertEqual(len(result), 11)
self.assertEqual(sorted(result), sorted(expected_pol_def_objs))
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
self.assertEqual('0',
resp.headers['X-Backend-Record-Storage-Policy-Index'])
# And if we specify a different idx we'll get objects for that policy
# and the X-Backend-Record-Storage-Policy-Index letting us know the
# policy for which these objects came from, if it differs from the
# policy stored in the DB.
req = Request.blank(
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
req.headers['X-Backend-Storage-Policy-Index'] = 1
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200)
result = [o for o in resp.body.split(b'\n') if o]
self.assertEqual(len(result), 10)
self.assertEqual(sorted(result), sorted(expected_pol_1_objs))
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
self.assertEqual('1',
resp.headers['X-Backend-Record-Storage-Policy-Index'])
# And an index that the broker doesn't have any objects for
req = Request.blank(
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
req.headers['X-Backend-Storage-Policy-Index'] = 2
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
result = [o for o in resp.body.split(b'\n') if o]
self.assertEqual(len(result), 0)
self.assertFalse(result)
self.assertIn('X-Backend-Storage-Policy-Index', resp.headers)
self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index'])
self.assertEqual('2',
resp.headers['X-Backend-Record-Storage-Policy-Index'])
# And an index that doesn't exist in POLICIES
req = Request.blank(
'/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
req.headers['X-Backend-Storage-Policy-Index'] = 3
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 400)
def test_GET_limit(self):
# make a container
req = Request.blank(

View File

@ -610,25 +610,30 @@ class TestContainerController(TestRingBase):
('a/c', {'X-Backend-Record-Type': 'auto'},
dict(states='listing')), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
{'X-Backend-Record-Type': 'auto'},
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='', end_marker='ham\x00', limit=str(limit),
states='listing')), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
{'X-Backend-Record-Type': 'auto'},
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='h', end_marker='pie\x00', states='listing',
limit=str(limit - len(sr_objs[0])))), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
{'X-Backend-Record-Type': 'auto'},
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing',
limit=str(limit - len(sr_objs[0] + sr_objs[1])))), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
{'X-Backend-Record-Type': 'auto'},
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='\xd1\xb0', end_marker='\xf0\x9f\x8c\xb4\x00',
states='listing',
limit=str(limit - len(sr_objs[0] + sr_objs[1]
+ sr_objs[2])))), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[4].name)),
{'X-Backend-Record-Type': 'auto'},
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='\xe2\xa8\x83', end_marker='', states='listing',
limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2]
+ sr_objs[3])))), # 200
@ -653,13 +658,19 @@ class TestContainerController(TestRingBase):
# path, headers, params
('a/c', {'X-Backend-Record-Type': 'auto'},
dict(states='listing')), # 200
(shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'},
(shard_ranges[0].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='', end_marker='ham\x00', limit=str(limit),
states='listing')), # 200
(shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'},
(shard_ranges[1].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='h', end_marker='pie\x00', states='listing',
limit=str(limit - len(sr_objs[0])))), # 200
(root_range.name, {'X-Backend-Record-Type': 'object'},
(root_range.name,
{'X-Backend-Record-Type': 'object',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='p', end_marker='',
limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200
]
@ -685,27 +696,32 @@ class TestContainerController(TestRingBase):
('a/c', {'X-Backend-Record-Type': 'auto'},
dict(states='listing', reverse='true', limit='')),
(wsgi_quote(str_to_wsgi(shard_ranges[4].name)),
{'X-Backend-Record-Type': 'auto'},
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='', end_marker='\xf0\x9f\x8c\xb4', states='listing',
reverse='true', limit=str(limit))), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
{'X-Backend-Record-Type': 'auto'},
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='\xf0\x9f\x8c\xb5', end_marker='\xe2\x98\x83',
states='listing', reverse='true',
limit=str(limit - len(sr_objs[4])))), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
{'X-Backend-Record-Type': 'auto'},
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='\xe2\x98\x84', end_marker='pie', states='listing',
reverse='true',
limit=str(limit - len(sr_objs[4] + sr_objs[3])))), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
{'X-Backend-Record-Type': 'auto'},
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='q', end_marker='ham', states='listing',
reverse='true',
limit=str(limit - len(sr_objs[4] + sr_objs[3]
+ sr_objs[2])))), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
{'X-Backend-Record-Type': 'auto'},
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='i', end_marker='', states='listing', reverse='true',
limit=str(limit - len(sr_objs[4] + sr_objs[3] + sr_objs[2]
+ sr_objs[1])))), # 200
@ -735,15 +751,18 @@ class TestContainerController(TestRingBase):
('a/c', {'X-Backend-Record-Type': 'auto'},
dict(limit=str(limit), states='listing')), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker='', end_marker='ham\x00', states='listing',
limit=str(limit))),
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker='h', end_marker='pie\x00', states='listing',
limit=str(limit - len(sr_objs[0])))),
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing',
limit=str(limit - len(sr_objs[0] + sr_objs[1])))),
]
@ -771,15 +790,18 @@ class TestContainerController(TestRingBase):
('a/c', {'X-Backend-Record-Type': 'auto'},
dict(marker=marker, states='listing')), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00',
states='listing', limit=str(limit))),
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00',
states='listing', limit=str(limit))),
(wsgi_quote(str_to_wsgi(shard_ranges[4].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker='\xe2\xa8\x83', end_marker='', states='listing',
limit=str(limit - len(sr_objs[3][2:])))),
]
@ -809,28 +831,34 @@ class TestContainerController(TestRingBase):
('a/c', {'X-Backend-Record-Type': 'auto'},
dict(end_marker=end_marker, states='listing')), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[0].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker='', end_marker='ham\x00', states='listing',
limit=str(limit))),
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
{'X-Backend-Record-Type': 'auto'}, # 404
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 404
dict(marker='h', end_marker='pie\x00', states='listing',
limit=str(limit - len(sr_objs[0])))),
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker='h', end_marker='pie\x00', states='listing',
limit=str(limit - len(sr_objs[0])))),
(wsgi_quote(str_to_wsgi(shard_ranges[2].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing',
limit=str(limit - len(sr_objs[0] + sr_objs[1])))),
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
{'X-Backend-Record-Type': 'auto'}, # 404
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 404
dict(marker='\xd1\xb0', end_marker=end_marker, states='listing',
limit=str(limit - len(sr_objs[0] + sr_objs[1]
+ sr_objs[2])))),
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker='\xd1\xb0', end_marker=end_marker, states='listing',
limit=str(limit - len(sr_objs[0] + sr_objs[1]
+ sr_objs[2])))),
@ -856,7 +884,8 @@ class TestContainerController(TestRingBase):
('a/c', {'X-Backend-Record-Type': 'auto'},
dict(prefix=prefix, states='listing')), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[1].name)),
{'X-Backend-Record-Type': 'auto'}, # 404
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 404
dict(prefix=prefix, marker='', end_marker='pie\x00',
states='listing', limit=str(limit))),
]
@ -877,7 +906,8 @@ class TestContainerController(TestRingBase):
dict(states='listing', limit=str(limit),
marker=marker, end_marker=end_marker)), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker=marker, end_marker=end_marker, states='listing',
limit=str(limit))),
]
@ -898,7 +928,8 @@ class TestContainerController(TestRingBase):
dict(marker=end_marker, reverse='true', end_marker=marker,
limit=str(limit), states='listing',)), # 200
(wsgi_quote(str_to_wsgi(shard_ranges[3].name)),
{'X-Backend-Record-Type': 'auto'}, # 200
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'}, # 200
dict(marker=end_marker, end_marker=marker, states='listing',
limit=str(limit), reverse='true')),
]
@ -1720,28 +1751,160 @@ class TestContainerController(TestRingBase):
('a/c', {'X-Backend-Record-Type': 'auto'},
dict(states='listing')), # 200
# get first shard objects
(shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'},
(shard_ranges[0].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='', end_marker='ham\x00', states='listing',
limit=str(limit))), # 200
# get second shard sub-shard ranges
(shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'},
(shard_ranges[1].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='h', end_marker='pie\x00', states='listing',
limit=str(limit - len(sr_objs[0])))),
# get first sub-shard objects
(sub_shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'},
(sub_shard_ranges[0].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='h', end_marker='juice\x00', states='listing',
limit=str(limit - len(sr_objs[0])))),
# get second sub-shard objects
(sub_shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'},
(sub_shard_ranges[1].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='j', end_marker='lemon\x00', states='listing',
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))),
# get remainder of first shard objects
(shard_ranges[1].name, {'X-Backend-Record-Type': 'object'},
(shard_ranges[1].name,
{'X-Backend-Record-Type': 'object',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='l', end_marker='pie\x00',
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] +
sub_sr_objs[1])))), # 200
# get third shard objects
(shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'},
(shard_ranges[2].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='p', end_marker='', states='listing',
limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200
]
expected_objects = (
sr_objs[0] + sub_sr_objs[0] + sub_sr_objs[1] +
sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):] + sr_objs[2])
resp = self._check_GET_shard_listing(
mock_responses, expected_objects, expected_requests)
# root object count will overridden by actual length of listing
self.check_response(resp, root_resp_hdrs)
@patch_policies([
StoragePolicy(0, 'zero', True, object_ring=FakeRing()),
StoragePolicy(1, 'one', False, object_ring=FakeRing())
])
def test_GET_sharded_container_sharding_shard_mixed_policies(self):
# scenario: one shard is in process of sharding, shards have different
# policy than root, expect listing to always request root policy index
shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', ''))
shard_ranges = [
ShardRange('.shards_a/c_' + upper, Timestamp.now(), lower, upper)
for lower, upper in shard_bounds]
sr_dicts = [dict(sr) for sr in shard_ranges]
sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges]
shard_resp_hdrs = [
{'X-Backend-Sharding-State': 'unsharded',
'X-Container-Object-Count': len(sr_objs[i]),
'X-Container-Bytes-Used':
sum([obj['bytes'] for obj in sr_objs[i]]),
'X-Container-Meta-Flavour': 'flavour%d' % i,
'X-Backend-Storage-Policy-Index': 1,
'X-Backend-Record-Storage-Policy-Index': 0}
for i in range(3)]
shard_1_shard_resp_hdrs = dict(shard_resp_hdrs[1])
shard_1_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard'
# second shard is sharding and has cleaved two out of three sub shards
shard_resp_hdrs[1]['X-Backend-Sharding-State'] = 'sharding'
sub_shard_bounds = (('ham', 'juice'), ('juice', 'lemon'))
sub_shard_ranges = [
ShardRange('a/c_sub_' + upper, Timestamp.now(), lower, upper)
for lower, upper in sub_shard_bounds]
sub_sr_dicts = [dict(sr) for sr in sub_shard_ranges]
sub_sr_objs = [self._make_shard_objects(sr) for sr in sub_shard_ranges]
sub_shard_resp_hdrs = [
{'X-Backend-Sharding-State': 'unsharded',
'X-Container-Object-Count': len(sub_sr_objs[i]),
'X-Container-Bytes-Used':
sum([obj['bytes'] for obj in sub_sr_objs[i]]),
'X-Container-Meta-Flavour': 'flavour%d' % i,
'X-Backend-Storage-Policy-Index': 1,
'X-Backend-Record-Storage-Policy-Index': 0}
for i in range(2)]
all_objects = []
for objects in sr_objs:
all_objects.extend(objects)
size_all_objects = sum([obj['bytes'] for obj in all_objects])
num_all_objects = len(all_objects)
limit = CONTAINER_LISTING_LIMIT
root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded',
'X-Backend-Timestamp': '99',
'X-Container-Object-Count': num_all_objects,
'X-Container-Bytes-Used': size_all_objects,
'X-Container-Meta-Flavour': 'peach',
'X-Backend-Storage-Policy-Index': 0}
root_shard_resp_hdrs = dict(root_resp_hdrs)
root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard'
mock_responses = [
# status, body, headers
(200, sr_dicts, root_shard_resp_hdrs),
(200, sr_objs[0], shard_resp_hdrs[0]),
(200, sub_sr_dicts + [sr_dicts[1]], shard_1_shard_resp_hdrs),
(200, sub_sr_objs[0], sub_shard_resp_hdrs[0]),
(200, sub_sr_objs[1], sub_shard_resp_hdrs[1]),
(200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):],
shard_resp_hdrs[1]),
(200, sr_objs[2], shard_resp_hdrs[2])
]
# NB marker always advances to last object name
expected_requests = [
# get root shard ranges
('a/c', {'X-Backend-Record-Type': 'auto'},
dict(states='listing')), # 200
# get first shard objects
(shard_ranges[0].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='', end_marker='ham\x00', states='listing',
limit=str(limit))), # 200
# get second shard sub-shard ranges
(shard_ranges[1].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='h', end_marker='pie\x00', states='listing',
limit=str(limit - len(sr_objs[0])))),
# get first sub-shard objects
(sub_shard_ranges[0].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='h', end_marker='juice\x00', states='listing',
limit=str(limit - len(sr_objs[0])))),
# get second sub-shard objects
(sub_shard_ranges[1].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='j', end_marker='lemon\x00', states='listing',
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))),
# get remainder of second shard objects
(shard_ranges[1].name,
{'X-Backend-Record-Type': 'object',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='l', end_marker='pie\x00',
limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] +
sub_sr_objs[1])))), # 200
# get third shard objects
(shard_ranges[2].name,
{'X-Backend-Record-Type': 'auto',
'X-Backend-Storage-Policy-Index': '0'},
dict(marker='p', end_marker='', states='listing',
limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200
]
@ -1927,6 +2090,88 @@ class TestContainerController(TestRingBase):
('delete', 'shard-listing/a/c', None, None)],
self.memcache.calls)
def test_get_from_shards_add_root_spi(self):
self._setup_shard_range_stubs()
shard_resp = mock.MagicMock(status_int=204, headers={})
def mock_get_container_listing(self_, req, *args, **kargs):
captured_hdrs.update(req.headers)
return None, shard_resp
# header in response -> header added to request
captured_hdrs = {}
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
resp = mock.MagicMock(body=self._stub_shards_dump,
headers=self.root_resp_hdrs,
request=req)
resp.headers['X-Backend-Storage-Policy-Index'] = '0'
with mock.patch('swift.proxy.controllers.container.'
'ContainerController._get_container_listing',
mock_get_container_listing):
controller_cls, d = self.app.get_controller(req)
controller = controller_cls(self.app, **d)
controller._get_from_shards(req, resp)
self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs)
self.assertEqual(
captured_hdrs['X-Backend-Storage-Policy-Index'], '0')
captured_hdrs = {}
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
resp = mock.MagicMock(body=self._stub_shards_dump,
headers=self.root_resp_hdrs,
request=req)
resp.headers['X-Backend-Storage-Policy-Index'] = '1'
with mock.patch('swift.proxy.controllers.container.'
'ContainerController._get_container_listing',
mock_get_container_listing):
controller_cls, d = self.app.get_controller(req)
controller = controller_cls(self.app, **d)
controller._get_from_shards(req, resp)
self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs)
self.assertEqual(
captured_hdrs['X-Backend-Storage-Policy-Index'], '1')
# header not added to request if not root request
captured_hdrs = {}
req = Request.blank('/v1/a/c',
environ={
'REQUEST_METHOD': 'GET',
'swift.shard_listing_history': [('a', 'c')]}
)
resp = mock.MagicMock(body=self._stub_shards_dump,
headers=self.root_resp_hdrs,
request=req)
resp.headers['X-Backend-Storage-Policy-Index'] = '0'
with mock.patch('swift.proxy.controllers.container.'
'ContainerController._get_container_listing',
mock_get_container_listing):
controller_cls, d = self.app.get_controller(req)
controller = controller_cls(self.app, **d)
controller._get_from_shards(req, resp)
self.assertNotIn('X-Backend-Storage-Policy-Index', captured_hdrs)
# existing X-Backend-Storage-Policy-Index in request is respected
captured_hdrs = {}
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
req.headers['X-Backend-Storage-Policy-Index'] = '0'
resp = mock.MagicMock(body=self._stub_shards_dump,
headers=self.root_resp_hdrs,
request=req)
resp.headers['X-Backend-Storage-Policy-Index'] = '1'
with mock.patch('swift.proxy.controllers.container.'
'ContainerController._get_container_listing',
mock_get_container_listing):
controller_cls, d = self.app.get_controller(req)
controller = controller_cls(self.app, **d)
controller._get_from_shards(req, resp)
self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs)
self.assertEqual(
captured_hdrs['X-Backend-Storage-Policy-Index'], '0')
def test_GET_shard_ranges(self):
self._setup_shard_range_stubs()
# expect shard ranges cache time to be default value of 600