swift/test/unit/common/test_request_helpers.py
Tim Burke 2a6dfae2f3 Allow direct and internal clients to use the replication network
A new header `X-Backend-Use-Replication-Network` is added; if true, use
the replication network instead of the client-data-path network.

Several background daemons are updated to use the replication network:

  * account-reaper
  * container-reconciler
  * container-sharder
  * container-sync
  * object-expirer

Note that if container-sync is being used to sync data within the same
cluster, the replication network will only be used when communicating
with the "source" container; the "destination" traffic will continue to
use the configured realm endpoint.

The direct and internal client APIs still default to using the
client-data-path network; this maintains backwards compatibility for
external tools written against them.

UpgradeImpact
=============

Until recently, servers configured with

  replication_server = true

would only handle REPLICATE (and, in the case of object servers, SSYNC)
requests, and would respond 405 Method Not Allowed to other requests.
When upgrading from Swift 2.25.0 or earlier, remove the config option
and restart services prior to upgrade to avoid a flood of background
daemon errors in logs.

Note that some background daemons find work by querying Swift rather
than walking local drives that should be available on the replication
network:

  * container-reconciler
  * object-expirer

Previosuly these may have been configured without access to the
replication network; ensure they have access before upgrading.

Closes-Bug: #1883302
Related-Bug: #1446873
Related-Change: Ica2b41a52d11cb10c94fa8ad780a201318c4fc87
Change-Id: Ieef534bf5d5fb53602e875b51c15ef565882fbff
2020-08-04 21:22:04 +00:00

566 lines
24 KiB
Python

# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for swift.common.request_helpers"""
import unittest
from swift.common.swob import Request, HTTPException, HeaderKeyDict
from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY
from swift.common import request_helpers as rh
from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX
from test.unit import patch_policies
from test.unit.common.test_utils import FakeResponse
server_types = ['account', 'container', 'object']
class TestRequestHelpers(unittest.TestCase):
def test_constrain_req_limit(self):
req = Request.blank('')
self.assertEqual(10, rh.constrain_req_limit(req, 10))
req = Request.blank('', query_string='limit=1')
self.assertEqual(1, rh.constrain_req_limit(req, 10))
req = Request.blank('', query_string='limit=1.0')
self.assertEqual(10, rh.constrain_req_limit(req, 10))
req = Request.blank('', query_string='limit=11')
with self.assertRaises(HTTPException) as raised:
rh.constrain_req_limit(req, 10)
self.assertEqual(raised.exception.status_int, 412)
def test_is_user_meta(self):
m_type = 'meta'
for st in server_types:
self.assertTrue(rh.is_user_meta(st, 'x-%s-%s-foo' % (st, m_type)))
self.assertFalse(rh.is_user_meta(st, 'x-%s-%s-' % (st, m_type)))
self.assertFalse(rh.is_user_meta(st, 'x-%s-%sfoo' % (st, m_type)))
def test_is_sys_meta(self):
m_type = 'sysmeta'
for st in server_types:
self.assertTrue(rh.is_sys_meta(st, 'x-%s-%s-foo' % (st, m_type)))
self.assertFalse(rh.is_sys_meta(st, 'x-%s-%s-' % (st, m_type)))
self.assertFalse(rh.is_sys_meta(st, 'x-%s-%sfoo' % (st, m_type)))
def test_is_sys_or_user_meta(self):
m_types = ['sysmeta', 'meta']
for mt in m_types:
for st in server_types:
self.assertTrue(rh.is_sys_or_user_meta(
st, 'x-%s-%s-foo' % (st, mt)))
self.assertFalse(rh.is_sys_or_user_meta(
st, 'x-%s-%s-' % (st, mt)))
self.assertFalse(rh.is_sys_or_user_meta(
st, 'x-%s-%sfoo' % (st, mt)))
def test_strip_sys_meta_prefix(self):
mt = 'sysmeta'
for st in server_types:
self.assertEqual(rh.strip_sys_meta_prefix(
st, 'x-%s-%s-a' % (st, mt)), 'a')
mt = 'not-sysmeta'
for st in server_types:
with self.assertRaises(ValueError):
rh.strip_sys_meta_prefix(st, 'x-%s-%s-a' % (st, mt))
def test_strip_user_meta_prefix(self):
mt = 'meta'
for st in server_types:
self.assertEqual(rh.strip_user_meta_prefix(
st, 'x-%s-%s-a' % (st, mt)), 'a')
mt = 'not-meta'
for st in server_types:
with self.assertRaises(ValueError):
rh.strip_sys_meta_prefix(st, 'x-%s-%s-a' % (st, mt))
def test_is_object_transient_sysmeta(self):
self.assertTrue(rh.is_object_transient_sysmeta(
'x-object-transient-sysmeta-foo'))
self.assertFalse(rh.is_object_transient_sysmeta(
'x-object-transient-sysmeta-'))
self.assertFalse(rh.is_object_transient_sysmeta(
'x-object-meatmeta-foo'))
def test_strip_object_transient_sysmeta_prefix(self):
mt = 'object-transient-sysmeta'
self.assertEqual(rh.strip_object_transient_sysmeta_prefix(
'x-%s-a' % mt), 'a')
mt = 'object-sysmeta-transient'
with self.assertRaises(ValueError):
rh.strip_object_transient_sysmeta_prefix('x-%s-a' % mt)
def test_remove_items(self):
src = {'a': 'b',
'c': 'd'}
test = lambda x: x == 'a'
rem = rh.remove_items(src, test)
self.assertEqual(src, {'c': 'd'})
self.assertEqual(rem, {'a': 'b'})
def test_copy_header_subset(self):
src = {'a': 'b',
'c': 'd'}
from_req = Request.blank('/path', environ={}, headers=src)
to_req = Request.blank('/path', {})
test = lambda x: x.lower() == 'a'
rh.copy_header_subset(from_req, to_req, test)
self.assertTrue('A' in to_req.headers)
self.assertEqual(to_req.headers['A'], 'b')
self.assertFalse('c' in to_req.headers)
self.assertFalse('C' in to_req.headers)
def test_get_ip_port(self):
node = {
'ip': '1.2.3.4',
'port': 6000,
'replication_ip': '5.6.7.8',
'replication_port': 7000,
}
self.assertEqual(('1.2.3.4', 6000), rh.get_ip_port(node, {}))
self.assertEqual(('5.6.7.8', 7000), rh.get_ip_port(node, {
rh.USE_REPLICATION_NETWORK_HEADER: 'true'}))
self.assertEqual(('1.2.3.4', 6000), rh.get_ip_port(node, {
rh.USE_REPLICATION_NETWORK_HEADER: 'false'}))
@patch_policies(with_ec_default=True)
def test_get_name_and_placement_object_req(self):
path = '/device/part/account/container/object'
req = Request.blank(path, headers={
'X-Backend-Storage-Policy-Index': '0'})
device, part, account, container, obj, policy = \
rh.get_name_and_placement(req, 5, 5, True)
self.assertEqual(device, 'device')
self.assertEqual(part, 'part')
self.assertEqual(account, 'account')
self.assertEqual(container, 'container')
self.assertEqual(obj, 'object')
self.assertEqual(policy, POLICIES[0])
self.assertEqual(policy.policy_type, EC_POLICY)
req.headers['X-Backend-Storage-Policy-Index'] = 1
device, part, account, container, obj, policy = \
rh.get_name_and_placement(req, 5, 5, True)
self.assertEqual(device, 'device')
self.assertEqual(part, 'part')
self.assertEqual(account, 'account')
self.assertEqual(container, 'container')
self.assertEqual(obj, 'object')
self.assertEqual(policy, POLICIES[1])
self.assertEqual(policy.policy_type, REPL_POLICY)
req.headers['X-Backend-Storage-Policy-Index'] = 'foo'
with self.assertRaises(HTTPException) as raised:
device, part, account, container, obj, policy = \
rh.get_name_and_placement(req, 5, 5, True)
e = raised.exception
self.assertEqual(e.status_int, 503)
self.assertEqual(str(e), '503 Service Unavailable')
self.assertEqual(e.body, b"No policy with index foo")
@patch_policies(with_ec_default=True)
def test_get_name_and_placement_object_replication(self):
# yup, suffixes are sent '-'.joined in the path
path = '/device/part/012-345-678-9ab-cde'
req = Request.blank(path, headers={
'X-Backend-Storage-Policy-Index': '0'})
device, partition, suffix_parts, policy = \
rh.get_name_and_placement(req, 2, 3, True)
self.assertEqual(device, 'device')
self.assertEqual(partition, 'part')
self.assertEqual(suffix_parts, '012-345-678-9ab-cde')
self.assertEqual(policy, POLICIES[0])
self.assertEqual(policy.policy_type, EC_POLICY)
path = '/device/part'
req = Request.blank(path, headers={
'X-Backend-Storage-Policy-Index': '1'})
device, partition, suffix_parts, policy = \
rh.get_name_and_placement(req, 2, 3, True)
self.assertEqual(device, 'device')
self.assertEqual(partition, 'part')
self.assertIsNone(suffix_parts) # false-y
self.assertEqual(policy, POLICIES[1])
self.assertEqual(policy.policy_type, REPL_POLICY)
path = '/device/part/' # with a trailing slash
req = Request.blank(path, headers={
'X-Backend-Storage-Policy-Index': '1'})
device, partition, suffix_parts, policy = \
rh.get_name_and_placement(req, 2, 3, True)
self.assertEqual(device, 'device')
self.assertEqual(partition, 'part')
self.assertEqual(suffix_parts, '') # still false-y
self.assertEqual(policy, POLICIES[1])
self.assertEqual(policy.policy_type, REPL_POLICY)
def test_validate_internal_name(self):
self.assertIsNone(rh._validate_internal_name('foo'))
self.assertIsNone(rh._validate_internal_name(
rh.get_reserved_name('foo')))
self.assertIsNone(rh._validate_internal_name(
rh.get_reserved_name('foo', 'bar')))
self.assertIsNone(rh._validate_internal_name(''))
self.assertIsNone(rh._validate_internal_name(rh.RESERVED))
def test_invalid_reserved_name(self):
with self.assertRaises(HTTPException) as raised:
rh._validate_internal_name('foo' + rh.RESERVED)
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace name")
def test_validate_internal_account(self):
self.assertIsNone(rh.validate_internal_account('AUTH_foo'))
self.assertIsNone(rh.validate_internal_account(
rh.get_reserved_name('AUTH_foo')))
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_account('AUTH_foo' + rh.RESERVED)
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace account")
def test_validate_internal_container(self):
self.assertIsNone(rh.validate_internal_container('AUTH_foo', 'bar'))
self.assertIsNone(rh.validate_internal_container(
rh.get_reserved_name('AUTH_foo'), 'bar'))
self.assertIsNone(rh.validate_internal_container(
'foo', rh.get_reserved_name('bar')))
self.assertIsNone(rh.validate_internal_container(
rh.get_reserved_name('AUTH_foo'), rh.get_reserved_name('bar')))
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_container('AUTH_foo' + rh.RESERVED, 'bar')
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace account")
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_container('AUTH_foo', 'bar' + rh.RESERVED)
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace container")
# These should always be operating on split_path outputs so this
# shouldn't really be an issue, but just in case...
for acct in ('', None):
with self.assertRaises(ValueError) as raised:
rh.validate_internal_container(
acct, 'bar')
self.assertEqual(raised.exception.args[0], 'Account is required')
def test_validate_internal_object(self):
self.assertIsNone(rh.validate_internal_obj('AUTH_foo', 'bar', 'baz'))
self.assertIsNone(rh.validate_internal_obj(
rh.get_reserved_name('AUTH_foo'), 'bar', 'baz'))
for acct in ('AUTH_foo', rh.get_reserved_name('AUTH_foo')):
self.assertIsNone(rh.validate_internal_obj(
acct,
rh.get_reserved_name('bar'),
rh.get_reserved_name('baz')))
for acct in ('AUTH_foo', rh.get_reserved_name('AUTH_foo')):
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_obj(
acct, 'bar', rh.get_reserved_name('baz'))
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace object "
b"in user-namespace container")
for acct in ('AUTH_foo', rh.get_reserved_name('AUTH_foo')):
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_obj(
acct, rh.get_reserved_name('bar'), 'baz')
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid user-namespace object "
b"in reserved-namespace container")
# These should always be operating on split_path outputs so this
# shouldn't really be an issue, but just in case...
for acct in ('', None):
with self.assertRaises(ValueError) as raised:
rh.validate_internal_obj(
acct, 'bar', 'baz')
self.assertEqual(raised.exception.args[0], 'Account is required')
for cont in ('', None):
with self.assertRaises(ValueError) as raised:
rh.validate_internal_obj(
'AUTH_foo', cont, 'baz')
self.assertEqual(raised.exception.args[0], 'Container is required')
def test_invalid_names_in_system_accounts(self):
self.assertIsNone(rh.validate_internal_obj(
AUTO_CREATE_ACCOUNT_PREFIX + 'system_account', 'foo',
'crazy%stown' % rh.RESERVED))
def test_invalid_reserved_names(self):
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_obj('AUTH_foo' + rh.RESERVED, 'bar', 'baz')
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace account")
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_obj('AUTH_foo', 'bar' + rh.RESERVED, 'baz')
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace container")
with self.assertRaises(HTTPException) as raised:
rh.validate_internal_obj('AUTH_foo', 'bar', 'baz' + rh.RESERVED)
e = raised.exception
self.assertEqual(e.status_int, 400)
self.assertEqual(str(e), '400 Bad Request')
self.assertEqual(e.body, b"Invalid reserved-namespace object")
def test_get_reserved_name(self):
expectations = {
tuple(): rh.RESERVED,
('',): rh.RESERVED,
('foo',): rh.RESERVED + 'foo',
('foo', 'bar'): rh.RESERVED + 'foo' + rh.RESERVED + 'bar',
('foo', ''): rh.RESERVED + 'foo' + rh.RESERVED,
('', ''): rh.RESERVED * 2,
}
failures = []
for parts, expected in expectations.items():
name = rh.get_reserved_name(*parts)
if name != expected:
failures.append('get given %r expected %r != %r' % (
parts, expected, name))
if failures:
self.fail('Unexpected reults:\n' + '\n'.join(failures))
def test_invalid_get_reserved_name(self):
self.assertRaises(ValueError)
with self.assertRaises(ValueError) as ctx:
rh.get_reserved_name('foo', rh.RESERVED + 'bar', 'baz')
self.assertEqual(str(ctx.exception),
'Invalid reserved part in components')
def test_split_reserved_name(self):
expectations = {
rh.RESERVED: ('',),
rh.RESERVED + 'foo': ('foo',),
rh.RESERVED + 'foo' + rh.RESERVED + 'bar': ('foo', 'bar'),
rh.RESERVED + 'foo' + rh.RESERVED: ('foo', ''),
rh.RESERVED * 2: ('', ''),
}
failures = []
for name, expected in expectations.items():
parts = rh.split_reserved_name(name)
if tuple(parts) != expected:
failures.append('split given %r expected %r != %r' % (
name, expected, parts))
if failures:
self.fail('Unexpected reults:\n' + '\n'.join(failures))
def test_invalid_split_reserved_name(self):
self.assertRaises(ValueError)
with self.assertRaises(ValueError) as ctx:
rh.split_reserved_name('foo')
self.assertEqual(str(ctx.exception),
'Invalid reserved name')
class TestHTTPResponseToDocumentIters(unittest.TestCase):
def test_200(self):
fr = FakeResponse(
200,
{'Content-Length': '10', 'Content-Type': 'application/lunch'},
b'sandwiches')
doc_iters = rh.http_response_to_document_iters(fr)
first_byte, last_byte, length, headers, body = next(doc_iters)
self.assertEqual(first_byte, 0)
self.assertEqual(last_byte, 9)
self.assertEqual(length, 10)
header_dict = HeaderKeyDict(headers)
self.assertEqual(header_dict.get('Content-Length'), '10')
self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
self.assertEqual(body.read(), b'sandwiches')
self.assertRaises(StopIteration, next, doc_iters)
fr = FakeResponse(
200,
{'Transfer-Encoding': 'chunked',
'Content-Type': 'application/lunch'},
b'sandwiches')
doc_iters = rh.http_response_to_document_iters(fr)
first_byte, last_byte, length, headers, body = next(doc_iters)
self.assertEqual(first_byte, 0)
self.assertIsNone(last_byte)
self.assertIsNone(length)
header_dict = HeaderKeyDict(headers)
self.assertEqual(header_dict.get('Transfer-Encoding'), 'chunked')
self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
self.assertEqual(body.read(), b'sandwiches')
self.assertRaises(StopIteration, next, doc_iters)
def test_206_single_range(self):
fr = FakeResponse(
206,
{'Content-Length': '8', 'Content-Type': 'application/lunch',
'Content-Range': 'bytes 1-8/10'},
b'andwiche')
doc_iters = rh.http_response_to_document_iters(fr)
first_byte, last_byte, length, headers, body = next(doc_iters)
self.assertEqual(first_byte, 1)
self.assertEqual(last_byte, 8)
self.assertEqual(length, 10)
header_dict = HeaderKeyDict(headers)
self.assertEqual(header_dict.get('Content-Length'), '8')
self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
self.assertEqual(body.read(), b'andwiche')
self.assertRaises(StopIteration, next, doc_iters)
# Chunked response should be treated in the same way as non-chunked one
fr = FakeResponse(
206,
{'Transfer-Encoding': 'chunked',
'Content-Type': 'application/lunch',
'Content-Range': 'bytes 1-8/10'},
b'andwiche')
doc_iters = rh.http_response_to_document_iters(fr)
first_byte, last_byte, length, headers, body = next(doc_iters)
self.assertEqual(first_byte, 1)
self.assertEqual(last_byte, 8)
self.assertEqual(length, 10)
header_dict = HeaderKeyDict(headers)
self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
self.assertEqual(body.read(), b'andwiche')
self.assertRaises(StopIteration, next, doc_iters)
def test_206_multiple_ranges(self):
fr = FakeResponse(
206,
{'Content-Type': 'multipart/byteranges; boundary=asdfasdfasdf'},
(b"--asdfasdfasdf\r\n"
b"Content-Type: application/lunch\r\n"
b"Content-Range: bytes 0-3/10\r\n"
b"\r\n"
b"sand\r\n"
b"--asdfasdfasdf\r\n"
b"Content-Type: application/lunch\r\n"
b"Content-Range: bytes 6-9/10\r\n"
b"\r\n"
b"ches\r\n"
b"--asdfasdfasdf--"))
doc_iters = rh.http_response_to_document_iters(fr)
first_byte, last_byte, length, headers, body = next(doc_iters)
self.assertEqual(first_byte, 0)
self.assertEqual(last_byte, 3)
self.assertEqual(length, 10)
header_dict = HeaderKeyDict(headers)
self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
self.assertEqual(body.read(), b'sand')
first_byte, last_byte, length, headers, body = next(doc_iters)
self.assertEqual(first_byte, 6)
self.assertEqual(last_byte, 9)
self.assertEqual(length, 10)
header_dict = HeaderKeyDict(headers)
self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
self.assertEqual(body.read(), b'ches')
self.assertRaises(StopIteration, next, doc_iters)
def test_update_etag_is_at_header(self):
# start with no existing X-Backend-Etag-Is-At
req = Request.blank('/v/a/c/o')
rh.update_etag_is_at_header(req, 'X-Object-Sysmeta-My-Etag')
self.assertEqual('X-Object-Sysmeta-My-Etag',
req.headers['X-Backend-Etag-Is-At'])
# add another alternate
rh.update_etag_is_at_header(req, 'X-Object-Sysmeta-Ec-Etag')
self.assertEqual('X-Object-Sysmeta-My-Etag,X-Object-Sysmeta-Ec-Etag',
req.headers['X-Backend-Etag-Is-At'])
with self.assertRaises(ValueError) as cm:
rh.update_etag_is_at_header(req, 'X-Object-Sysmeta-,-Bad')
self.assertEqual('Header name must not contain commas',
cm.exception.args[0])
def test_resolve_etag_is_at_header(self):
def do_test():
req = Request.blank('/v/a/c/o')
# ok to have no X-Backend-Etag-Is-At
self.assertIsNone(rh.resolve_etag_is_at_header(req, metadata))
# ok to have no matching metadata
req.headers['X-Backend-Etag-Is-At'] = 'X-Not-There'
self.assertIsNone(rh.resolve_etag_is_at_header(req, metadata))
# selects from metadata
req.headers['X-Backend-Etag-Is-At'] = 'X-Object-Sysmeta-Ec-Etag'
self.assertEqual('an etag value',
rh.resolve_etag_is_at_header(req, metadata))
req.headers['X-Backend-Etag-Is-At'] = 'X-Object-Sysmeta-My-Etag'
self.assertEqual('another etag value',
rh.resolve_etag_is_at_header(req, metadata))
# first in list takes precedence
req.headers['X-Backend-Etag-Is-At'] = \
'X-Object-Sysmeta-My-Etag,X-Object-Sysmeta-Ec-Etag'
self.assertEqual('another etag value',
rh.resolve_etag_is_at_header(req, metadata))
# non-existent alternates are passed over
req.headers['X-Backend-Etag-Is-At'] = \
'X-Bogus,X-Object-Sysmeta-My-Etag,X-Object-Sysmeta-Ec-Etag'
self.assertEqual('another etag value',
rh.resolve_etag_is_at_header(req, metadata))
# spaces in list are ok
alts = 'X-Foo, X-Object-Sysmeta-My-Etag , X-Object-Sysmeta-Ec-Etag'
req.headers['X-Backend-Etag-Is-At'] = alts
self.assertEqual('another etag value',
rh.resolve_etag_is_at_header(req, metadata))
# lower case in list is ok
alts = alts.lower()
req.headers['X-Backend-Etag-Is-At'] = alts
self.assertEqual('another etag value',
rh.resolve_etag_is_at_header(req, metadata))
# upper case in list is ok
alts = alts.upper()
req.headers['X-Backend-Etag-Is-At'] = alts
self.assertEqual('another etag value',
rh.resolve_etag_is_at_header(req, metadata))
metadata = {'X-Object-Sysmeta-Ec-Etag': 'an etag value',
'X-Object-Sysmeta-My-Etag': 'another etag value'}
do_test()
metadata = dict((k.lower(), v) for k, v in metadata.items())
do_test()
metadata = dict((k.upper(), v) for k, v in metadata.items())
do_test()