diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index b89f8e6c45..759bd7cd25 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -592,8 +592,8 @@ use = egg:swift#s3api # you don't expect. # s3_acl = false # -# Specify a host name of your Swift cluster. This enables virtual-hosted style -# requests. +# Specify a (comma-separated) list of host names for your Swift cluster. +# This enables virtual-hosted style requests. # storage_domain = # # Enable pipeline order check for SLO, s3token, authtoken, keystoneauth diff --git a/swift/common/middleware/s3api/s3api.py b/swift/common/middleware/s3api/s3api.py index 9c225c53b9..bf1d9d7b9f 100644 --- a/swift/common/middleware/s3api/s3api.py +++ b/swift/common/middleware/s3api/s3api.py @@ -263,7 +263,8 @@ class S3ApiMiddleware(object): wsgi_conf.get('multi_delete_concurrency', 2)) self.conf.s3_acl = config_true_value( wsgi_conf.get('s3_acl', False)) - self.conf.storage_domain = wsgi_conf.get('storage_domain', '') + self.conf.storage_domains = list_from_csv( + wsgi_conf.get('storage_domain', '')) self.conf.auth_pipeline_check = config_true_value( wsgi_conf.get('auth_pipeline_check', True)) self.conf.max_upload_part_num = config_positive_int_value( diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index d4c33512de..e52a2d4ca8 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -603,25 +603,25 @@ class S3Request(swob.Request): return 'AWSAccessKeyId' in self.params def _parse_host(self): - storage_domain = self.conf.storage_domain - if not storage_domain: + if not self.conf.storage_domains: return None - if not storage_domain.startswith('.'): - storage_domain = '.' + storage_domain - if 'HTTP_HOST' in self.environ: given_domain = self.environ['HTTP_HOST'] elif 'SERVER_NAME' in self.environ: given_domain = self.environ['SERVER_NAME'] else: return None - port = '' if ':' in given_domain: given_domain, port = given_domain.rsplit(':', 1) - if given_domain.endswith(storage_domain): - return given_domain[:-len(storage_domain)] + + for storage_domain in self.conf.storage_domains: + if not storage_domain.startswith('.'): + storage_domain = '.' + storage_domain + + if given_domain.endswith(storage_domain): + return given_domain[:-len(storage_domain)] return None diff --git a/swift/common/middleware/s3api/utils.py b/swift/common/middleware/s3api/utils.py index 41f89192f9..4c8a4fd670 100644 --- a/swift/common/middleware/s3api/utils.py +++ b/swift/common/middleware/s3api/utils.py @@ -153,7 +153,7 @@ def mktime(timestamp_str, time_format='%Y-%m-%dT%H:%M:%S'): class Config(dict): DEFAULTS = { - 'storage_domain': '', + 'storage_domains': [], 'location': 'us-east-1', 'force_swift_request_proxy_log': False, 'dns_compliant_bucket_names': True, diff --git a/test/unit/common/middleware/s3api/test_s3api.py b/test/unit/common/middleware/s3api/test_s3api.py index 73e974e9c4..b490a2537a 100644 --- a/test/unit/common/middleware/s3api/test_s3api.py +++ b/test/unit/common/middleware/s3api/test_s3api.py @@ -107,7 +107,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): # will be short-circuited # check all defaults - expected = Config() + expected = dict(Config()) expected.update({ 'auth_pipeline_check': True, 'check_bucket_owner': False, @@ -126,7 +126,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): # check all non-defaults are loaded conf = { - 'storage_domain': 'somewhere', + 'storage_domain': 'somewhere,some.other.where', 'location': 'us-west-1', 'force_swift_request_proxy_log': True, 'dns_compliant_bucket_names': False, @@ -148,6 +148,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): s3api = S3ApiMiddleware(None, conf) conf['cors_preflight_allow_origin'] = \ conf['cors_preflight_allow_origin'].split(',') + conf['storage_domains'] = conf.pop('storage_domain').split(',') self.assertEqual(conf, s3api.conf) # test allow_origin list with a '*' fails. diff --git a/test/unit/common/middleware/s3api/test_s3request.py b/test/unit/common/middleware/s3api/test_s3request.py index 056bc32eb0..0d4ee27f5c 100644 --- a/test/unit/common/middleware/s3api/test_s3request.py +++ b/test/unit/common/middleware/s3api/test_s3request.py @@ -651,18 +651,51 @@ class TestRequest(S3ApiTestCase): # Virtual hosted-style req = Request.blank('/', environ=environ, headers=headers) sigv2_req = S3Request( - req.environ, conf=Config({'storage_domain': 's3.test.com'})) + req.environ, conf=Config({'storage_domains': ['s3.test.com']})) uri = sigv2_req._canonical_uri() self.assertEqual(uri, '/bucket1/') self.assertEqual(req.environ['PATH_INFO'], '/') req = Request.blank('/obj1', environ=environ, headers=headers) sigv2_req = S3Request( - req.environ, conf=Config({'storage_domain': 's3.test.com'})) + req.environ, conf=Config({'storage_domains': ['s3.test.com']})) uri = sigv2_req._canonical_uri() self.assertEqual(uri, '/bucket1/obj1') self.assertEqual(req.environ['PATH_INFO'], '/obj1') + req = Request.blank('/obj2', environ=environ, headers=headers) + sigv2_req = S3Request( + req.environ, conf=Config({ + 'storage_domains': ['alternate.domain', 's3.test.com']})) + uri = sigv2_req._canonical_uri() + self.assertEqual(uri, '/bucket1/obj2') + self.assertEqual(req.environ['PATH_INFO'], '/obj2') + + # Now check the other storage_domain + environ = { + 'HTTP_HOST': 'bucket1.alternate.domain', + 'REQUEST_METHOD': 'GET'} + req = Request.blank('/obj2', environ=environ, headers=headers) + sigv2_req = S3Request( + req.environ, conf=Config({ + 'storage_domains': ['alternate.domain', 's3.test.com']})) + uri = sigv2_req._canonical_uri() + self.assertEqual(uri, '/bucket1/obj2') + self.assertEqual(req.environ['PATH_INFO'], '/obj2') + + # Non existent storage_domain means we can't find the container + environ = { + 'HTTP_HOST': 'bucket1.incorrect.domain', + 'REQUEST_METHOD': 'GET'} + req = Request.blank('/obj2', environ=environ, headers=headers) + sigv2_req = S3Request( + req.environ, conf=Config({ + 'storage_domains': ['alternate.domain', 's3.test.com']})) + uri = sigv2_req._canonical_uri() + # uo oh, no bucket + self.assertEqual(uri, '/obj2') + self.assertEqual(sigv2_req.container_name, 'obj2') + environ = { 'HTTP_HOST': 's3.test.com', 'REQUEST_METHOD': 'GET'} @@ -701,7 +734,7 @@ class TestRequest(S3ApiTestCase): 'X-Amz-Date': x_amz_date} # Virtual hosted-style - self.s3api.conf.storage_domain = 's3.test.com' + self.s3api.conf.storage_domains = ['s3.test.com'] req = Request.blank('/', environ=environ, headers=headers) sigv4_req = SigV4Request(req.environ) uri = sigv4_req._canonical_uri() @@ -721,7 +754,7 @@ class TestRequest(S3ApiTestCase): 'REQUEST_METHOD': 'GET'} # Path-style - self.s3api.conf.storage_domain = '' + self.s3api.conf.storage_domains = [] req = Request.blank('/', environ=environ, headers=headers) sigv4_req = SigV4Request(req.environ) uri = sigv4_req._canonical_uri() @@ -749,7 +782,7 @@ class TestRequest(S3ApiTestCase): 'bWq2s1WEIj+Ydj0vQ697zp+IXMU='), }) sigv2_req = S3Request(req.environ, conf=Config({ - 'storage_domain': 's3.amazonaws.com'})) + 'storage_domains': ['s3.amazonaws.com']})) expected_sts = b'\n'.join([ b'GET', b'', @@ -769,7 +802,7 @@ class TestRequest(S3ApiTestCase): 'MyyxeRY7whkBe+bq8fHCL/2kKUg='), }) sigv2_req = S3Request(req.environ, conf=Config({ - 'storage_domain': 's3.amazonaws.com'})) + 'storage_domains': ['s3.amazonaws.com']})) expected_sts = b'\n'.join([ b'PUT', b'', @@ -790,7 +823,7 @@ class TestRequest(S3ApiTestCase): 'htDYFYduRNen8P9ZfE/s9SuKy0U='), }) sigv2_req = S3Request(req.environ, conf=Config({ - 'storage_domain': 's3.amazonaws.com'})) + 'storage_domains': ['s3.amazonaws.com']})) expected_sts = b'\n'.join([ b'GET', b'', @@ -819,7 +852,7 @@ class TestRequest(S3ApiTestCase): 'bWq2s1WEIj+Ydj0vQ697zp+IXMU='), }) sigv2_req = S3Request(req.environ, Config({ - 'storage_domain': 's3.amazonaws.com'})) + 'storage_domains': ['s3.amazonaws.com']})) # This is a failure case with utf-8 non-ascii multi-bytes charactor # but we expect to return just False instead of exceptions self.assertFalse(sigv2_req.check_signature( @@ -837,7 +870,7 @@ class TestRequest(S3ApiTestCase): 'X-Amz-Date': amz_date_header }) sigv4_req = SigV4Request( - req.environ, Config({'storage_domain': 's3.amazonaws.com'})) + req.environ, Config({'storage_domains': ['s3.amazonaws.com']})) self.assertFalse(sigv4_req.check_signature( u'\u30c9\u30e9\u30b4\u30f3')) @@ -858,7 +891,7 @@ class TestRequest(S3ApiTestCase): 'X-Amz-Date': '20210104T102623Z'} # Virtual hosted-style - self.s3api.conf.storage_domain = 's3.test.com' + self.s3api.conf.storage_domains = ['s3.test.com'] req = Request.blank('/', environ=environ, headers=headers) sigv4_req = SigV4Request(req.environ) self.assertTrue( @@ -868,7 +901,7 @@ class TestRequest(S3ApiTestCase): @patch.object(S3Request, '_validate_dates', lambda *a: None) def test_check_sigv4_req_zero_content_length_sha256(self): # Virtual hosted-style - self.s3api.conf.storage_domain = 's3.test.com' + self.s3api.conf.storage_domains = ['s3.test.com'] # bad sha256 environ = { diff --git a/test/unit/common/middleware/s3api/test_utils.py b/test/unit/common/middleware/s3api/test_utils.py index 3b7cb959e2..ad6fc119fa 100644 --- a/test/unit/common/middleware/s3api/test_utils.py +++ b/test/unit/common/middleware/s3api/test_utils.py @@ -133,7 +133,7 @@ class TestS3ApiUtils(unittest.TestCase): class TestConfig(unittest.TestCase): def _assert_defaults(self, conf): - self.assertEqual('', conf.storage_domain) + self.assertEqual([], conf.storage_domains) self.assertEqual('us-east-1', conf.location) self.assertFalse(conf.force_swift_request_proxy_log) self.assertTrue(conf.dns_compliant_bucket_names) @@ -146,7 +146,7 @@ class TestConfig(unittest.TestCase): # deliberately brittle so new defaults will need to be added to test conf = utils.Config() self._assert_defaults(conf) - del conf.storage_domain + del conf.storage_domains del conf.location del conf.force_swift_request_proxy_log del conf.dns_compliant_bucket_names