swift/test/unit/common/middleware/test_helpers.py
Clay Gerrard 451ae26a8b Improve FakeSwift Backend-Ignore-Range support
In keeping with the trend as of late, this change makes FakeSwift behave
more reliably like a real Swift backend.

Swift backend object server's grew support for ignoring Range request
headers when fetching SLO manifests in Jan-2020, and FakeSwift learned
how to mimic the real behavior in Jul-2022.  This change unifies the
implementation details with a request_helper and consolidates the
behavior in FakeSwift.  It also makes the modern object-server behavior
the default.

Between 2020 and 2022 there was arguably some utility defaulting to
legacy behavior, but in 2023 as we endeavor to refactor the SLO
implementation and extend it's tests: a reliable FakeSwift is paramount.

Since most of the existing tests for SLO's behavior responding to Range
requests did not reliably assert behavior across new and old swift this
change selects the most relevant tests to legacy behavior and has them
opt-in to can_ignore_range = False, while the others merely have their
backend request asserts cleaned-up to match the backend request pattern
you would expect in a production environment that's upgraded in the last
3 years.  Additional technical investment may be required to ensure
older clusters can upgrade proxies before object servers w/o tracebacks
until the upgrade finishes; however it appears the existing code is
sufficiently robust despite the lack of explicit multi-inheritance
testing like was done for the legacy manifest format change in Nov-2016
(N.B. unlike rolling upgrade bugs, data is forever).

Related-Change-Id: I4ff2a178d0456e7e37d561109ef57dd0d92cbd4e
Related-Change-Id: If3861e5b9c4f17ab3b82ea16673ddb29d07820a1
Related-Change-Id: Ia6ad32354105515560b005cea750aa64a88c96f9

Change-Id: I7ebfd557b9c8ec25498c628fcf0695cd52ad78d6
2023-09-06 13:06:33 +01:00

511 lines
22 KiB
Python

# -*- coding: utf-8 -*-
# Copyright (c) 2023 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from swift.common.swob import Request, HTTPOk, HTTPNotFound, HTTPCreated
from swift.common import request_helpers as rh
from test.unit.common.middleware.helpers import FakeSwift
class TestFakeSwift(unittest.TestCase):
def test_not_registered(self):
swift = FakeSwift()
def do_test(method):
req = Request.blank('/v1/a/c/o')
req.method = method
with self.assertRaises(KeyError):
req.get_response(swift)
do_test('GET')
do_test('HEAD')
do_test('POST')
do_test('PUT')
do_test('DELETE')
def test_GET_registered(self):
# verify that a single registered GET response is sufficient to handle
# GETs and HEADS, with and without query strings
swift = FakeSwift()
swift.register('GET', '/v1/a/c/o', HTTPOk, {'X-Foo': 'Bar'}, b'stuff')
req = Request.blank('/v1/a/c/o')
req.method = 'GET'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'X-Foo': 'Bar'},
resp.headers)
self.assertEqual(b'stuff', resp.body)
self.assertEqual(1, swift.call_count)
self.assertEqual(('GET', '/v1/a/c/o'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'GET'
req.query_string = 'p=q'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'X-Foo': 'Bar'},
resp.headers)
self.assertEqual(b'stuff', resp.body)
self.assertEqual(2, swift.call_count)
self.assertEqual(('GET', '/v1/a/c/o?p=q'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'HEAD'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Type': 'text/html; charset=UTF-8',
'X-Foo': 'Bar'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(3, swift.call_count)
self.assertEqual(('HEAD', '/v1/a/c/o'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'HEAD'
req.query_string = 'p=q'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Type': 'text/html; charset=UTF-8',
'X-Foo': 'Bar'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(4, swift.call_count)
self.assertEqual(('HEAD', '/v1/a/c/o?p=q'), swift.calls[-1])
def test_GET_registered_with_query_string(self):
# verify that a single registered GET response is sufficient to handle
# GETs and HEADS, with and without query strings
swift = FakeSwift()
swift.register('GET', '/v1/a/c/o?p=q', HTTPOk,
{'X-Foo': 'Bar'}, b'stuff')
req = Request.blank('/v1/a/c/o')
req.method = 'GET'
with self.assertRaises(KeyError):
resp = req.get_response(swift)
req.query_string = 'p=q'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'X-Foo': 'Bar'},
resp.headers)
self.assertEqual(b'stuff', resp.body)
self.assertEqual(1, swift.call_count)
self.assertEqual(('GET', '/v1/a/c/o?p=q'), swift.calls[-1])
def test_GET_and_HEAD_registered(self):
# verify that a registered HEAD response will be preferred over GET for
# HEAD request
swift = FakeSwift()
swift.register('GET', '/v1/a/c/o', HTTPOk, {'X-Foo': 'Bar'}, b'stuff')
swift.register('HEAD', '/v1/a/c/o', HTTPNotFound, {}, b'')
req = Request.blank('/v1/a/c/o')
req.method = 'GET'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'X-Foo': 'Bar'},
resp.headers)
self.assertEqual(b'stuff', resp.body)
self.assertEqual(1, swift.call_count)
self.assertEqual(('GET', '/v1/a/c/o'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'HEAD'
resp = req.get_response(swift)
self.assertEqual(404, resp.status_int)
self.assertEqual({'Content-Type': 'text/html; charset=UTF-8'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(2, swift.call_count)
self.assertEqual(('HEAD', '/v1/a/c/o'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'HEAD'
req.query_string = 'p=q'
resp = req.get_response(swift)
self.assertEqual(404, resp.status_int)
self.assertEqual({'Content-Type': 'text/html; charset=UTF-8'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(3, swift.call_count)
self.assertEqual(('HEAD', '/v1/a/c/o?p=q'), swift.calls[-1])
def test_PUT_uploaded(self):
# verify an uploaded object is sufficient to handle GETs and HEADS,
# with and without query strings
swift = FakeSwift()
swift.register('PUT', '/v1/a/c/o', HTTPCreated, {}, None)
req = Request.blank('/v1/a/c/o', body=b'stuff')
req.method = 'PUT'
resp = req.get_response(swift)
self.assertEqual(201, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Etag': 'c13d88cb4cb02003daedb8a84e5d272a',
'Content-Type': 'text/html; charset=UTF-8'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(1, swift.call_count)
self.assertEqual(('PUT', '/v1/a/c/o'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'GET'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'Host': 'localhost:80'},
resp.headers)
self.assertEqual(b'stuff', resp.body)
self.assertEqual(2, swift.call_count)
self.assertEqual(('GET', '/v1/a/c/o'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'GET'
req.query_string = 'p=q'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'Host': 'localhost:80'},
resp.headers)
self.assertEqual(b'stuff', resp.body)
self.assertEqual(3, swift.call_count)
self.assertEqual(('GET', '/v1/a/c/o?p=q'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'HEAD'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'Host': 'localhost:80'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(4, swift.call_count)
self.assertEqual(('HEAD', '/v1/a/c/o'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'HEAD'
req.query_string = 'p=q'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'Host': 'localhost:80'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(5, swift.call_count)
self.assertEqual(('HEAD', '/v1/a/c/o?p=q'), swift.calls[-1])
def test_PUT_uploaded_with_query_string(self):
# verify an uploaded object with query string is sufficient to handle
# GETs and HEADS, with and without query strings
swift = FakeSwift()
swift.register('PUT', '/v1/a/c/o', HTTPCreated, {}, None)
req = Request.blank('/v1/a/c/o', body=b'stuff')
req.method = 'PUT'
req.query_string = 'multipart-manifest=put'
resp = req.get_response(swift)
self.assertEqual(201, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Etag': 'c13d88cb4cb02003daedb8a84e5d272a',
'Content-Type': 'text/html; charset=UTF-8'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(1, swift.call_count)
self.assertEqual(('PUT', '/v1/a/c/o?multipart-manifest=put'),
swift.calls[-1])
# note: query string is not included in uploaded key
self.assertEqual(
{'/v1/a/c/o': ({'Host': 'localhost:80',
'Content-Length': '5'},
b'stuff')},
swift.uploaded)
req = Request.blank('/v1/a/c/o')
req.method = 'GET'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'Host': 'localhost:80'},
resp.headers)
self.assertEqual(b'stuff', resp.body)
self.assertEqual(2, swift.call_count)
self.assertEqual(('GET', '/v1/a/c/o'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'GET'
req.query_string = 'p=q' # note: differs from PUT query string
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'Host': 'localhost:80'},
resp.headers)
self.assertEqual(b'stuff', resp.body)
self.assertEqual(3, swift.call_count)
self.assertEqual(('GET', '/v1/a/c/o?p=q'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'HEAD'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'Host': 'localhost:80'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(4, swift.call_count)
self.assertEqual(('HEAD', '/v1/a/c/o'), swift.calls[-1])
req = Request.blank('/v1/a/c/o')
req.method = 'HEAD'
req.query_string = 'p=q'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'Host': 'localhost:80'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(5, swift.call_count)
self.assertEqual(('HEAD', '/v1/a/c/o?p=q'), swift.calls[-1])
def test_PUT_POST(self):
# verify an uploaded object is updated by a POST
swift = FakeSwift()
swift.register('PUT', '/v1/a/c/o', HTTPCreated, {}, None)
# Note: the POST must be registered
swift.register('POST', '/v1/a/c/o', HTTPCreated, {}, None)
req = Request.blank('/v1/a/c/o', body=b'stuff',
headers={'X-Object-Meta-Foo': 'Bar'})
req.method = 'PUT'
resp = req.get_response(swift)
self.assertEqual(201, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Etag': 'c13d88cb4cb02003daedb8a84e5d272a',
'Content-Type': 'text/html; charset=UTF-8'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(1, swift.call_count)
self.assertEqual(('PUT', '/v1/a/c/o'), swift.calls[-1])
self.assertEqual(
{'/v1/a/c/o': ({'Host': 'localhost:80',
'Content-Length': '5',
'X-Object-Meta-Foo': 'Bar'},
b'stuff')},
swift.uploaded)
# POST should update the uploaded object
req = Request.blank('/v1/a/c/o', body=b'stuff',
headers={'X-Object-Meta-Foo': 'Baz'})
req.method = 'POST'
resp = req.get_response(swift)
self.assertEqual(201, resp.status_int)
self.assertEqual({'Content-Type': 'text/html; charset=UTF-8'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(2, swift.call_count)
self.assertEqual(('POST', '/v1/a/c/o'), swift.calls[-1])
self.assertEqual(
{'/v1/a/c/o': ({'Host': 'localhost:80',
'Content-Length': '5',
'X-Object-Meta-Foo': 'Baz'},
b'stuff')},
swift.uploaded)
def test_PUT_with_query_string_POST(self):
# verify an uploaded object with query string is updated by a POST
swift = FakeSwift()
swift.register('PUT', '/v1/a/c/o', HTTPCreated, {}, None)
# Note: the POST must be registered
swift.register('POST', '/v1/a/c/o', HTTPCreated, {}, None)
req = Request.blank('/v1/a/c/o', body=b'stuff',
headers={'X-Object-Meta-Foo': 'Bar'})
req.method = 'PUT'
req.query_string = 'p=q'
resp = req.get_response(swift)
self.assertEqual(201, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Etag': 'c13d88cb4cb02003daedb8a84e5d272a',
'Content-Type': 'text/html; charset=UTF-8'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(1, swift.call_count)
self.assertEqual(('PUT', '/v1/a/c/o?p=q'), swift.calls[-1])
# note: query string is not included in uploaded key
self.assertEqual(
{'/v1/a/c/o': ({'Host': 'localhost:80',
'Content-Length': '5',
'X-Object-Meta-Foo': 'Bar'},
b'stuff')},
swift.uploaded)
# POST without query string should update the uploaded object
req = Request.blank('/v1/a/c/o', body=b'stuff',
headers={'X-Object-Meta-Foo': 'Baz'})
req.method = 'POST'
resp = req.get_response(swift)
self.assertEqual(201, resp.status_int)
self.assertEqual({'Content-Type': 'text/html; charset=UTF-8'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(2, swift.call_count)
self.assertEqual(('POST', '/v1/a/c/o'), swift.calls[-1])
self.assertEqual(
{'/v1/a/c/o': ({'Host': 'localhost:80',
'Content-Length': '5',
'X-Object-Meta-Foo': 'Baz'},
b'stuff')},
swift.uploaded)
# POST with different query string should update the uploaded object
req = Request.blank('/v1/a/c/o', body=b'stuff',
headers={'X-Object-Meta-Foo': 'Bof'})
req.method = 'POST'
req.query_string = 'x=y'
resp = req.get_response(swift)
self.assertEqual(201, resp.status_int)
self.assertEqual({'Content-Type': 'text/html; charset=UTF-8'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(3, swift.call_count)
self.assertEqual(('POST', '/v1/a/c/o?x=y'), swift.calls[-1])
self.assertEqual(
{'/v1/a/c/o': ({'Host': 'localhost:80',
'Content-Length': '5',
'X-Object-Meta-Foo': 'Bof'},
b'stuff')},
swift.uploaded)
req = Request.blank('/v1/a/c/o')
req.method = 'GET'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Content-Type': 'text/html; charset=UTF-8',
'Host': 'localhost:80',
'X-Object-Meta-Foo': 'Bof'},
resp.headers)
self.assertEqual(b'stuff', resp.body)
self.assertEqual(4, swift.call_count)
self.assertEqual(('GET', '/v1/a/c/o'), swift.calls[-1])
def test_GET_registered_overrides_uploaded(self):
swift = FakeSwift()
swift.register('PUT', '/v1/a/c/o', HTTPCreated, {}, None)
swift.register('GET', '/v1/a/c/o', HTTPOk, {}, b'not stuff')
req = Request.blank('/v1/a/c/o', body=b'stuff',
headers={'X-Object-Meta-Foo': 'Bar'})
req.method = 'PUT'
resp = req.get_response(swift)
self.assertEqual(201, resp.status_int)
self.assertEqual({'Content-Length': '5',
'Etag': 'c13d88cb4cb02003daedb8a84e5d272a',
'Content-Type': 'text/html; charset=UTF-8'},
resp.headers)
self.assertEqual(b'', resp.body)
self.assertEqual(1, swift.call_count)
self.assertEqual(('PUT', '/v1/a/c/o'), swift.calls[-1])
self.assertEqual(
{'/v1/a/c/o': ({'Host': 'localhost:80',
'Content-Length': '5',
'X-Object-Meta-Foo': 'Bar'},
b'stuff')},
swift.uploaded)
req = Request.blank('/v1/a/c/o')
req.method = 'GET'
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual({'Content-Length': '9',
'Content-Type': 'text/html; charset=UTF-8'},
resp.headers)
self.assertEqual(b'not stuff', resp.body)
self.assertEqual(2, swift.call_count)
self.assertEqual(('GET', '/v1/a/c/o'), swift.calls[-1])
def test_range(self):
swift = FakeSwift()
swift.register('GET', '/v1/a/c/o', HTTPOk, {}, b'stuff')
req = Request.blank('/v1/a/c/o', headers={'Range': 'bytes=0-2'})
resp = req.get_response(swift)
self.assertEqual(206, resp.status_int)
self.assertEqual(b'stu', resp.body)
self.assertEqual('bytes 0-2/5', resp.headers['Content-Range'])
self.assertEqual('bytes=0-2', req.headers.get('Range'))
self.assertEqual('bytes=0-2',
swift.calls_with_headers[-1].headers.get('Range'))
def test_range_ignore_range_header(self):
swift = FakeSwift()
swift.register('GET', '/v1/a/c/o', HTTPOk, {
# the value of the matching header doesn't matter
'X-Object-Sysmeta-Magic': 'False'
}, b'stuff')
req = Request.blank('/v1/a/c/o', headers={'Range': 'bytes=0-2'})
rh.update_ignore_range_header(req, 'X-Object-Sysmeta-Magic')
resp = req.get_response(swift)
self.assertEqual(200, resp.status_int)
self.assertEqual(b'stuff', resp.body)
self.assertNotIn('Content-Range', resp.headers)
self.assertEqual('bytes=0-2', req.headers.get('Range'))
self.assertEqual('bytes=0-2',
swift.calls_with_headers[-1].headers.get('Range'))
def test_range_ignore_range_header_old_swift(self):
swift = FakeSwift()
swift.can_ignore_range = False
swift.register('GET', '/v1/a/c/o', HTTPOk, {
# the value of the matching header doesn't matter
'X-Object-Sysmeta-Magic': 'False'
}, b'stuff')
req = Request.blank('/v1/a/c/o', headers={'Range': 'bytes=0-2'})
rh.update_ignore_range_header(req, 'X-Object-Sysmeta-Magic')
resp = req.get_response(swift)
self.assertEqual(206, resp.status_int)
self.assertEqual(b'stu', resp.body)
self.assertEqual('bytes 0-2/5', resp.headers['Content-Range'])
self.assertEqual('bytes=0-2', req.headers.get('Range'))
self.assertEqual('bytes=0-2',
swift.calls_with_headers[-1].headers.get('Range'))
def test_range_ignore_range_header_ignored(self):
swift = FakeSwift()
# range is only ignored if registered response has matching metadata
swift.register('GET', '/v1/a/c/o', HTTPOk, {}, b'stuff')
req = Request.blank('/v1/a/c/o', headers={'Range': 'bytes=0-2'})
rh.update_ignore_range_header(req, 'X-Object-Sysmeta-Magic')
resp = req.get_response(swift)
self.assertEqual(206, resp.status_int)
self.assertEqual(b'stu', resp.body)
self.assertEqual('bytes 0-2/5', resp.headers['Content-Range'])
self.assertEqual('bytes=0-2', req.headers.get('Range'))
self.assertEqual('bytes=0-2',
swift.calls_with_headers[-1].headers.get('Range'))