openstack
/
swift
Code Issues Proposed changes

Add stats for shard range cache hits/misses 

Add some stats to indicate when the proxy fails to get shard ranges
into memcache for object updates.

Change-Id: I63332b3794a1a61a512a70aad240459fec4810f3
This commit is contained in:
Alistair Coles 2021-12-17 14:24:02 +00:00 committed by Tim Burke
parent ebf75f6e2a
commit 40e0f577a9
3 changed files with 148 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
swift/proxy/controllers/base.py
View File

@ -2351,7 +2351,7 @@ class Controller(object):
headers = {'X-Backend-Record-Type': 'shard'}
listing, response = self._get_container_listing(
req, account, container, headers=headers, params=params)
return self._parse_shard_ranges(req, listing, response)
return self._parse_shard_ranges(req, listing, response), response
def _get_update_shard(self, req, account, container, obj):
"""
@ -2370,8 +2370,10 @@ class Controller(object):
"""
if not self.app.recheck_updating_shard_ranges:
# caching is disabled; fall back to old behavior
shard_ranges = self._get_shard_ranges(
shard_ranges, response = self._get_shard_ranges(
req, account, container, states='updating', includes=obj)
self.app.logger.increment(
'shard_updating.backend.%s' % response.status_int)
if not shard_ranges:
return None
return shard_ranges[0]
@ -2383,14 +2385,18 @@ class Controller(object):
cached_ranges = infocache.get(cache_key)
if cached_ranges is None and memcache:
cached_ranges = memcache.get(cache_key)
self.app.logger.increment('shard_updating.cache.%s'
% ('hit' if cached_ranges else 'miss'))
if cached_ranges:
shard_ranges = [
ShardRange.from_dict(shard_range)
for shard_range in cached_ranges]
else:
shard_ranges = self._get_shard_ranges(
shard_ranges, response = self._get_shard_ranges(
req, account, container, states='updating')
self.app.logger.increment(
'shard_updating.backend.%s' % response.status_int)
if shard_ranges:
cached_ranges = [dict(sr) for sr in shard_ranges]
# went to disk; cache it

18
test/unit/proxy/controllers/test_base.py
View File

@ -1328,7 +1328,8 @@ class TestFuncs(BaseTest):
body_iter=iter([b'', json.dumps(shard_ranges).encode('ascii')]),
headers=resp_headers
) as fake_conn:
actual = base._get_shard_ranges(req, 'a', 'c')
actual, resp = base._get_shard_ranges(req, 'a', 'c')
self.assertEqual(200, resp.status_int)
# account info
captured = fake_conn.requests
@ -1359,7 +1360,8 @@ class TestFuncs(BaseTest):
json.dumps(shard_ranges[1:2]).encode('ascii')]),
headers=resp_headers
) as fake_conn:
actual = base._get_shard_ranges(req, 'a', 'c', '1_test')
actual, resp = base._get_shard_ranges(req, 'a', 'c', '1_test')
self.assertEqual(200, resp.status_int)
# account info
captured = fake_conn.requests
@ -1383,7 +1385,8 @@ class TestFuncs(BaseTest):
headers = {'X-Backend-Record-Type': 'shard'}
with mocked_http_conn(200, 200, body_iter=iter([b'', body]),
headers=headers):
actual = base._get_shard_ranges(req, 'a', 'c', '1_test')
actual, resp = base._get_shard_ranges(req, 'a', 'c', '1_test')
self.assertEqual(200, resp.status_int)
self.assertIsNone(actual)
lines = self.app.logger.get_lines_for_level('error')
return lines
@ -1427,7 +1430,8 @@ class TestFuncs(BaseTest):
body = json.dumps([dict(sr)]).encode('ascii')
with mocked_http_conn(
200, 200, body_iter=iter([b'', body])):
actual = base._get_shard_ranges(req, 'a', 'c', '1_test')
actual, resp = base._get_shard_ranges(req, 'a', 'c', '1_test')
self.assertEqual(200, resp.status_int)
self.assertIsNone(actual)
error_lines = self.app.logger.get_lines_for_level('error')
self.assertIn('Failed to get shard ranges', error_lines[0])
@ -1444,7 +1448,8 @@ class TestFuncs(BaseTest):
with mocked_http_conn(
200, 200, body_iter=iter([b'', body]),
headers=headers):
actual = base._get_shard_ranges(req, 'a', 'c', '1_test')
actual, resp = base._get_shard_ranges(req, 'a', 'c', '1_test')
self.assertEqual(200, resp.status_int)
self.assertIsNone(actual)
error_lines = self.app.logger.get_lines_for_level('error')
self.assertIn('Failed to get shard ranges', error_lines[0])
@ -1456,7 +1461,8 @@ class TestFuncs(BaseTest):
base = Controller(self.app)
req = Request.blank('/v1/a/c/o', method='PUT')
with mocked_http_conn(200, 404, 404, 404):
actual = base._get_shard_ranges(req, 'a', 'c', '1_test')
actual, resp = base._get_shard_ranges(req, 'a', 'c', '1_test')
self.assertEqual(404, resp.status_int)
self.assertIsNone(actual)
self.assertFalse(self.app.logger.get_lines_for_level('error'))
warning_lines = self.app.logger.get_lines_for_level('warning')

197
test/unit/proxy/test_server.py
View File

@ -3911,6 +3911,25 @@ class TestReplicatedObjectController(
resp_headers['X-Backend-Sharding-State'] = 'unsharded'
do_test(resp_headers)
def _check_request(self, req, method, path, headers=None, params=None):
self.assertEqual(method, req['method'])
# caller can ignore leading path parts
self.assertTrue(req['path'].endswith(path),
'expected path to end with %s, it was %s' % (
path, req['path']))
headers = headers or {}
# caller can ignore some headers
for k, v in headers.items():
self.assertEqual(req['headers'][k], v,
'Expected %s but got %s for key %s' %
(v, req['headers'][k], k))
params = params or {}
req_params = dict(parse_qsl(req['qs'])) if req['qs'] else {}
for k, v in params.items():
self.assertEqual(req_params[k], v,
'Expected %s but got %s for key %s' %
(v, req_params[k], k))
@patch_policies([
StoragePolicy(0, 'zero', is_default=True, object_ring=FakeRing()),
StoragePolicy(1, 'one', object_ring=FakeRing()),
@ -3924,6 +3943,7 @@ class TestReplicatedObjectController(
self.app.recheck_updating_shard_ranges = 0
def do_test(method, sharding_state):
self.app.logger = debug_logger('proxy-ut') # clean capture state
req = Request.blank('/v1/a/c/o', {}, method=method, body='',
headers={'Content-Type': 'text/plain'})
@ -3942,33 +3962,18 @@ class TestReplicatedObjectController(
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
stats = self.app.logger.get_increment_counts()
self.assertEqual({'shard_updating.backend.200': 1}, stats)
backend_requests = fake_conn.requests
def check_request(req, method, path, headers=None, params=None):
self.assertEqual(method, req['method'])
# caller can ignore leading path parts
self.assertTrue(req['path'].endswith(path),
'expected path to end with %s, it was %s' % (
path, req['path']))
headers = headers or {}
# caller can ignore some headers
for k, v in headers.items():
self.assertEqual(req['headers'][k], v,
'Expected %s but got %s for key %s' %
(v, req['headers'][k], k))
params = params or {}
req_params = dict(parse_qsl(req['qs'])) if req['qs'] else {}
for k, v in params.items():
self.assertEqual(req_params[k], v,
'Expected %s but got %s for key %s' %
(v, req_params[k], k))
account_request = backend_requests[0]
check_request(account_request, method='HEAD', path='/sda/0/a')
self._check_request(
account_request, method='HEAD', path='/sda/0/a')
container_request = backend_requests[1]
check_request(container_request, method='HEAD', path='/sda/0/a/c')
self._check_request(
container_request, method='HEAD', path='/sda/0/a/c')
container_request_shard = backend_requests[2]
check_request(
self._check_request(
container_request_shard, method='GET', path='/sda/0/a/c',
params={'includes': 'o', 'states': 'updating'},
headers={'X-Backend-Record-Type': 'shard'})
@ -3991,7 +3996,7 @@ class TestReplicatedObjectController(
'X-Backend-Quoted-Container-Path': shard_range.name
},
}
check_request(request, **expectations)
self._check_request(request, **expectations)
expected = {}
for i, device in enumerate(['sda', 'sdb', 'sdc']):
@ -4018,6 +4023,7 @@ class TestReplicatedObjectController(
self.app.recheck_updating_shard_ranges = 3600
def do_test(method, sharding_state):
self.app.logger = debug_logger('proxy-ut') # clean capture state
req = Request.blank(
'/v1/a/c/o', {'swift.cache': FakeMemcache()},
method=method, body='', headers={'Content-Type': 'text/plain'})
@ -4045,33 +4051,19 @@ class TestReplicatedObjectController(
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
stats = self.app.logger.get_increment_counts()
self.assertEqual({'shard_updating.cache.miss': 1,
'shard_updating.backend.200': 1}, stats)
backend_requests = fake_conn.requests
def check_request(req, method, path, headers=None, params=None):
self.assertEqual(method, req['method'])
# caller can ignore leading path parts
self.assertTrue(req['path'].endswith(path),
'expected path to end with %s, it was %s' % (
path, req['path']))
headers = headers or {}
# caller can ignore some headers
for k, v in headers.items():
self.assertEqual(req['headers'][k], v,
'Expected %s but got %s for key %s' %
(v, req['headers'][k], k))
params = params or {}
req_params = dict(parse_qsl(req['qs'])) if req['qs'] else {}
for k, v in params.items():
self.assertEqual(req_params[k], v,
'Expected %s but got %s for key %s' %
(v, req_params[k], k))
account_request = backend_requests[0]
check_request(account_request, method='HEAD', path='/sda/0/a')
self._check_request(
account_request, method='HEAD', path='/sda/0/a')
container_request = backend_requests[1]
check_request(container_request, method='HEAD', path='/sda/0/a/c')
self._check_request(
container_request, method='HEAD', path='/sda/0/a/c')
container_request_shard = backend_requests[2]
check_request(
self._check_request(
container_request_shard, method='GET', path='/sda/0/a/c',
params={'states': 'updating'},
headers={'X-Backend-Record-Type': 'shard'})
@ -4102,7 +4094,7 @@ class TestReplicatedObjectController(
'X-Backend-Quoted-Container-Path': shard_ranges[1].name
},
}
check_request(request, **expectations)
self._check_request(request, **expectations)
expected = {}
for i, device in enumerate(['sda', 'sdb', 'sdc']):
@ -4129,6 +4121,7 @@ class TestReplicatedObjectController(
self.app.recheck_updating_shard_ranges = 3600
def do_test(method, sharding_state):
self.app.logger = debug_logger('proxy-ut') # clean capture state
shard_ranges = [
utils.ShardRange(
'.shards_a/c_not_used', utils.Timestamp.now(), '', 'l'),
@ -4156,31 +4149,16 @@ class TestReplicatedObjectController(
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
stats = self.app.logger.get_increment_counts()
self.assertEqual({'shard_updating.cache.hit': 1}, stats)
backend_requests = fake_conn.requests
def check_request(req, method, path, headers=None, params=None):
self.assertEqual(method, req['method'])
# caller can ignore leading path parts
self.assertTrue(req['path'].endswith(path),
'expected path to end with %s, it was %s' % (
path, req['path']))
headers = headers or {}
# caller can ignore some headers
for k, v in headers.items():
self.assertEqual(req['headers'][k], v,
'Expected %s but got %s for key %s' %
(v, req['headers'][k], k))
params = params or {}
req_params = dict(parse_qsl(req['qs'])) if req['qs'] else {}
for k, v in params.items():
self.assertEqual(req_params[k], v,
'Expected %s but got %s for key %s' %
(v, req_params[k], k))
account_request = backend_requests[0]
check_request(account_request, method='HEAD', path='/sda/0/a')
self._check_request(
account_request, method='HEAD', path='/sda/0/a')
container_request = backend_requests[1]
check_request(container_request, method='HEAD', path='/sda/0/a/c')
self._check_request(
container_request, method='HEAD', path='/sda/0/a/c')
# infocache gets populated from memcache
cache_key = 'shard-updating/a/c'
@ -4206,7 +4184,86 @@ class TestReplicatedObjectController(
'X-Backend-Quoted-Container-Path': shard_ranges[1].name
},
}
check_request(request, **expectations)
self._check_request(request, **expectations)
expected = {}
for i, device in enumerate(['sda', 'sdb', 'sdc']):
expected[device] = '10.0.0.%d:100%d' % (i, i)
self.assertEqual(container_headers, expected)
do_test('POST', 'sharding')
do_test('POST', 'sharded')
do_test('DELETE', 'sharding')
do_test('DELETE', 'sharded')
do_test('PUT', 'sharding')
do_test('PUT', 'sharded')
@patch_policies([
StoragePolicy(0, 'zero', is_default=True, object_ring=FakeRing()),
StoragePolicy(1, 'one', object_ring=FakeRing()),
])
def test_backend_headers_update_shard_container_errors(self):
# verify that update target reverts to root if get shard ranges fails
# reset the router post patch_policies
self.app.obj_controller_router = proxy_server.ObjectControllerRouter()
self.app.sort_nodes = lambda nodes, *args, **kwargs: nodes
self.app.recheck_updating_shard_ranges = 0
def do_test(method, sharding_state):
self.app.logger = debug_logger('proxy-ut') # clean capture state
req = Request.blank('/v1/a/c/o', {}, method=method, body='',
headers={'Content-Type': 'text/plain'})
# we want the container_info response to say policy index of 1 and
# sharding state, but we want shard range listings to fail
# acc HEAD, cont HEAD, cont shard GETs, obj POSTs
status_codes = (200, 200, 404, 404, 404, 202, 202, 202)
resp_headers = {'X-Backend-Storage-Policy-Index': 1,
'x-backend-sharding-state': sharding_state}
with mocked_http_conn(*status_codes,
headers=resp_headers) as fake_conn:
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
stats = self.app.logger.get_increment_counts()
self.assertEqual({'shard_updating.backend.404': 1}, stats)
backend_requests = fake_conn.requests
account_request = backend_requests[0]
self._check_request(
account_request, method='HEAD', path='/sda/0/a')
container_request = backend_requests[1]
self._check_request(
container_request, method='HEAD', path='/sda/0/a/c')
container_request_shard = backend_requests[2]
self._check_request(
container_request_shard, method='GET', path='/sda/0/a/c',
params={'includes': 'o', 'states': 'updating'},
headers={'X-Backend-Record-Type': 'shard'})
# infocache does not get populated from memcache
cache_key = 'shard-updating/a/c'
self.assertNotIn(cache_key, req.environ.get('swift.infocache'))
# make sure backend requests included expected container headers
container_headers = {}
for request in backend_requests[5:]:
req_headers = request['headers']
device = req_headers['x-container-device']
container_headers[device] = req_headers['x-container-host']
expectations = {
'method': method,
'path': '/0/a/c/o',
'headers': {
'X-Container-Partition': '0',
'Host': 'localhost:80',
'Referer': '%s http://localhost/v1/a/c/o' % method,
'X-Backend-Storage-Policy-Index': '1',
# X-Backend-Quoted-Container-Path is not sent
},
}
self._check_request(request, **expectations)
expected = {}
for i, device in enumerate(['sda', 'sdb', 'sdc']):