openstack/swift
Code Issues Proposed changes

Merge "s3api: Allow multiple storage domains"

Browse Source
This commit is contained in:
Zuul
2022-01-28 20:24:04 +00:00
committed by Gerrit Code Review
parent c46e987694 11d1022163
commit c1d2e661b1
7 changed files with 62 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
etc/proxy-server.conf-sample
View File

@@ -592,8 +592,8 @@ use = egg:swift#s3api
# you don't expect.
# s3_acl = false
#
# Specify a host name of your Swift cluster. This enables virtual-hosted style
# requests.
# Specify a (comma-separated) list of host names for your Swift cluster.
# This enables virtual-hosted style requests.
# storage_domain =
#
# Enable pipeline order check for SLO, s3token, authtoken, keystoneauth

3
swift/common/middleware/s3api/s3api.py
View File

@@ -263,7 +263,8 @@ class S3ApiMiddleware(object):
wsgi_conf.get('multi_delete_concurrency', 2))
self.conf.s3_acl = config_true_value(
wsgi_conf.get('s3_acl', False))
self.conf.storage_domain = wsgi_conf.get('storage_domain', '')
self.conf.storage_domains = list_from_csv(
wsgi_conf.get('storage_domain', ''))
self.conf.auth_pipeline_check = config_true_value(
wsgi_conf.get('auth_pipeline_check', True))
self.conf.max_upload_part_num = config_positive_int_value(

16
swift/common/middleware/s3api/s3request.py
View File

@@ -603,25 +603,25 @@ class S3Request(swob.Request):
return 'AWSAccessKeyId' in self.params
def _parse_host(self):
storage_domain = self.conf.storage_domain
if not storage_domain:
if not self.conf.storage_domains:
return None
if not storage_domain.startswith('.'):
storage_domain = '.' + storage_domain
if 'HTTP_HOST' in self.environ:
given_domain = self.environ['HTTP_HOST']
elif 'SERVER_NAME' in self.environ:
given_domain = self.environ['SERVER_NAME']
else:
return None
port = ''
if ':' in given_domain:
given_domain, port = given_domain.rsplit(':', 1)
if given_domain.endswith(storage_domain):
return given_domain[:-len(storage_domain)]
for storage_domain in self.conf.storage_domains:
if not storage_domain.startswith('.'):
storage_domain = '.' + storage_domain
if given_domain.endswith(storage_domain):
return given_domain[:-len(storage_domain)]
return None

2
swift/common/middleware/s3api/utils.py
View File

@@ -153,7 +153,7 @@ def mktime(timestamp_str, time_format='%Y-%m-%dT%H:%M:%S'):
class Config(dict):
DEFAULTS = {
'storage_domain': '',
'storage_domains': [],
'location': 'us-east-1',
'force_swift_request_proxy_log': False,
'dns_compliant_bucket_names': True,

5
test/unit/common/middleware/s3api/test_s3api.py
View File

@@ -107,7 +107,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
# will be short-circuited
# check all defaults
expected = Config()
expected = dict(Config())
expected.update({
'auth_pipeline_check': True,
'check_bucket_owner': False,
@@ -126,7 +126,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
# check all non-defaults are loaded
conf = {
'storage_domain': 'somewhere',
'storage_domain': 'somewhere,some.other.where',
'location': 'us-west-1',
'force_swift_request_proxy_log': True,
'dns_compliant_bucket_names': False,
@@ -148,6 +148,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
s3api = S3ApiMiddleware(None, conf)
conf['cors_preflight_allow_origin'] = \
conf['cors_preflight_allow_origin'].split(',')
conf['storage_domains'] = conf.pop('storage_domain').split(',')
self.assertEqual(conf, s3api.conf)
# test allow_origin list with a '*' fails.

55
test/unit/common/middleware/s3api/test_s3request.py
View File

@@ -651,18 +651,51 @@ class TestRequest(S3ApiTestCase):
# Virtual hosted-style
req = Request.blank('/', environ=environ, headers=headers)
sigv2_req = S3Request(
req.environ, conf=Config({'storage_domain': 's3.test.com'}))
req.environ, conf=Config({'storage_domains': ['s3.test.com']}))
uri = sigv2_req._canonical_uri()
self.assertEqual(uri, '/bucket1/')
self.assertEqual(req.environ['PATH_INFO'], '/')
req = Request.blank('/obj1', environ=environ, headers=headers)
sigv2_req = S3Request(
req.environ, conf=Config({'storage_domain': 's3.test.com'}))
req.environ, conf=Config({'storage_domains': ['s3.test.com']}))
uri = sigv2_req._canonical_uri()
self.assertEqual(uri, '/bucket1/obj1')
self.assertEqual(req.environ['PATH_INFO'], '/obj1')
req = Request.blank('/obj2', environ=environ, headers=headers)
sigv2_req = S3Request(
req.environ, conf=Config({
'storage_domains': ['alternate.domain', 's3.test.com']}))
uri = sigv2_req._canonical_uri()
self.assertEqual(uri, '/bucket1/obj2')
self.assertEqual(req.environ['PATH_INFO'], '/obj2')
# Now check the other storage_domain
environ = {
'HTTP_HOST': 'bucket1.alternate.domain',
'REQUEST_METHOD': 'GET'}
req = Request.blank('/obj2', environ=environ, headers=headers)
sigv2_req = S3Request(
req.environ, conf=Config({
'storage_domains': ['alternate.domain', 's3.test.com']}))
uri = sigv2_req._canonical_uri()
self.assertEqual(uri, '/bucket1/obj2')
self.assertEqual(req.environ['PATH_INFO'], '/obj2')
# Non existent storage_domain means we can't find the container
environ = {
'HTTP_HOST': 'bucket1.incorrect.domain',
'REQUEST_METHOD': 'GET'}
req = Request.blank('/obj2', environ=environ, headers=headers)
sigv2_req = S3Request(
req.environ, conf=Config({
'storage_domains': ['alternate.domain', 's3.test.com']}))
uri = sigv2_req._canonical_uri()
# uo oh, no bucket
self.assertEqual(uri, '/obj2')
self.assertEqual(sigv2_req.container_name, 'obj2')
environ = {
'HTTP_HOST': 's3.test.com',
'REQUEST_METHOD': 'GET'}
@@ -701,7 +734,7 @@ class TestRequest(S3ApiTestCase):
'X-Amz-Date': x_amz_date}
# Virtual hosted-style
self.s3api.conf.storage_domain = 's3.test.com'
self.s3api.conf.storage_domains = ['s3.test.com']
req = Request.blank('/', environ=environ, headers=headers)
sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri()
@@ -721,7 +754,7 @@ class TestRequest(S3ApiTestCase):
'REQUEST_METHOD': 'GET'}
# Path-style
self.s3api.conf.storage_domain = ''
self.s3api.conf.storage_domains = []
req = Request.blank('/', environ=environ, headers=headers)
sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri()
@@ -749,7 +782,7 @@ class TestRequest(S3ApiTestCase):
'bWq2s1WEIj+Ydj0vQ697zp+IXMU='),
})
sigv2_req = S3Request(req.environ, conf=Config({
'storage_domain': 's3.amazonaws.com'}))
'storage_domains': ['s3.amazonaws.com']}))
expected_sts = b'\n'.join([
b'GET',
b'',
@@ -769,7 +802,7 @@ class TestRequest(S3ApiTestCase):
'MyyxeRY7whkBe+bq8fHCL/2kKUg='),
})
sigv2_req = S3Request(req.environ, conf=Config({
'storage_domain': 's3.amazonaws.com'}))
'storage_domains': ['s3.amazonaws.com']}))
expected_sts = b'\n'.join([
b'PUT',
b'',
@@ -790,7 +823,7 @@ class TestRequest(S3ApiTestCase):
'htDYFYduRNen8P9ZfE/s9SuKy0U='),
})
sigv2_req = S3Request(req.environ, conf=Config({
'storage_domain': 's3.amazonaws.com'}))
'storage_domains': ['s3.amazonaws.com']}))
expected_sts = b'\n'.join([
b'GET',
b'',
@@ -819,7 +852,7 @@ class TestRequest(S3ApiTestCase):
'bWq2s1WEIj+Ydj0vQ697zp+IXMU='),
})
sigv2_req = S3Request(req.environ, Config({
'storage_domain': 's3.amazonaws.com'}))
'storage_domains': ['s3.amazonaws.com']}))
# This is a failure case with utf-8 non-ascii multi-bytes charactor
# but we expect to return just False instead of exceptions
self.assertFalse(sigv2_req.check_signature(
@@ -837,7 +870,7 @@ class TestRequest(S3ApiTestCase):
'X-Amz-Date': amz_date_header
})
sigv4_req = SigV4Request(
req.environ, Config({'storage_domain': 's3.amazonaws.com'}))
req.environ, Config({'storage_domains': ['s3.amazonaws.com']}))
self.assertFalse(sigv4_req.check_signature(
u'\u30c9\u30e9\u30b4\u30f3'))
@@ -858,7 +891,7 @@ class TestRequest(S3ApiTestCase):
'X-Amz-Date': '20210104T102623Z'}
# Virtual hosted-style
self.s3api.conf.storage_domain = 's3.test.com'
self.s3api.conf.storage_domains = ['s3.test.com']
req = Request.blank('/', environ=environ, headers=headers)
sigv4_req = SigV4Request(req.environ)
self.assertTrue(
@@ -868,7 +901,7 @@ class TestRequest(S3ApiTestCase):
@patch.object(S3Request, '_validate_dates', lambda *a: None)
def test_check_sigv4_req_zero_content_length_sha256(self):
# Virtual hosted-style
self.s3api.conf.storage_domain = 's3.test.com'
self.s3api.conf.storage_domains = ['s3.test.com']
# bad sha256
environ = {

4
test/unit/common/middleware/s3api/test_utils.py
View File

@@ -133,7 +133,7 @@ class TestS3ApiUtils(unittest.TestCase):
class TestConfig(unittest.TestCase):
def _assert_defaults(self, conf):
self.assertEqual('', conf.storage_domain)
self.assertEqual([], conf.storage_domains)
self.assertEqual('us-east-1', conf.location)
self.assertFalse(conf.force_swift_request_proxy_log)
self.assertTrue(conf.dns_compliant_bucket_names)
@@ -146,7 +146,7 @@ class TestConfig(unittest.TestCase):
# deliberately brittle so new defaults will need to be added to test
conf = utils.Config()
self._assert_defaults(conf)
del conf.storage_domain
del conf.storage_domains
del conf.location
del conf.force_swift_request_proxy_log
del conf.dns_compliant_bucket_names