Merge "s3api: Allow multiple storage domains"

etc/proxy-server.conf-sample

@@ -592,8 +592,8 @@ use = egg:swift#s3api
 # you don't expect.
 # s3_acl = false
 #
-# Specify a host name of your Swift cluster.  This enables virtual-hosted style
+# Specify a (comma-separated) list of host names for your Swift cluster.
-# requests.
+# This enables virtual-hosted style requests.
 # storage_domain =
 #
 # Enable pipeline order check for SLO, s3token, authtoken, keystoneauth

swift/common/middleware/s3api/s3api.py

@@ -263,7 +263,8 @@ class S3ApiMiddleware(object):
             wsgi_conf.get('multi_delete_concurrency', 2))
         self.conf.s3_acl = config_true_value(
             wsgi_conf.get('s3_acl', False))
-        self.conf.storage_domain = wsgi_conf.get('storage_domain', '')
+        self.conf.storage_domains = list_from_csv(
+            wsgi_conf.get('storage_domain', ''))
         self.conf.auth_pipeline_check = config_true_value(
             wsgi_conf.get('auth_pipeline_check', True))
         self.conf.max_upload_part_num = config_positive_int_value(

swift/common/middleware/s3api/s3request.py

@@ -603,23 +603,23 @@ class S3Request(swob.Request):
         return 'AWSAccessKeyId' in self.params
 
     def _parse_host(self):
-        storage_domain = self.conf.storage_domain
+        if not self.conf.storage_domains:
-        if not storage_domain:
             return None
 
-        if not storage_domain.startswith('.'):
-            storage_domain = '.' + storage_domain
-
         if 'HTTP_HOST' in self.environ:
             given_domain = self.environ['HTTP_HOST']
         elif 'SERVER_NAME' in self.environ:
             given_domain = self.environ['SERVER_NAME']
         else:
             return None
-
         port = ''
         if ':' in given_domain:
             given_domain, port = given_domain.rsplit(':', 1)
+
+        for storage_domain in self.conf.storage_domains:
+            if not storage_domain.startswith('.'):
+                storage_domain = '.' + storage_domain
+
             if given_domain.endswith(storage_domain):
                 return given_domain[:-len(storage_domain)]
 

swift/common/middleware/s3api/utils.py

@@ -153,7 +153,7 @@ def mktime(timestamp_str, time_format='%Y-%m-%dT%H:%M:%S'):
 
 class Config(dict):
     DEFAULTS = {
-        'storage_domain': '',
+        'storage_domains': [],
         'location': 'us-east-1',
         'force_swift_request_proxy_log': False,
         'dns_compliant_bucket_names': True,

test/unit/common/middleware/s3api/test_s3api.py

@@ -107,7 +107,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
         # will be short-circuited
 
         # check all defaults
-        expected = Config()
+        expected = dict(Config())
         expected.update({
             'auth_pipeline_check': True,
             'check_bucket_owner': False,
@@ -126,7 +126,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
 
         # check all non-defaults are loaded
         conf = {
-            'storage_domain': 'somewhere',
+            'storage_domain': 'somewhere,some.other.where',
             'location': 'us-west-1',
             'force_swift_request_proxy_log': True,
             'dns_compliant_bucket_names': False,
@@ -148,6 +148,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
         s3api = S3ApiMiddleware(None, conf)
         conf['cors_preflight_allow_origin'] = \
             conf['cors_preflight_allow_origin'].split(',')
+        conf['storage_domains'] = conf.pop('storage_domain').split(',')
         self.assertEqual(conf, s3api.conf)
 
         # test allow_origin list with a '*' fails.

test/unit/common/middleware/s3api/test_s3request.py

@@ -651,18 +651,51 @@ class TestRequest(S3ApiTestCase):
         # Virtual hosted-style
         req = Request.blank('/', environ=environ, headers=headers)
         sigv2_req = S3Request(
-            req.environ, conf=Config({'storage_domain': 's3.test.com'}))
+            req.environ, conf=Config({'storage_domains': ['s3.test.com']}))
         uri = sigv2_req._canonical_uri()
         self.assertEqual(uri, '/bucket1/')
         self.assertEqual(req.environ['PATH_INFO'], '/')
 
         req = Request.blank('/obj1', environ=environ, headers=headers)
         sigv2_req = S3Request(
-            req.environ, conf=Config({'storage_domain': 's3.test.com'}))
+            req.environ, conf=Config({'storage_domains': ['s3.test.com']}))
         uri = sigv2_req._canonical_uri()
         self.assertEqual(uri, '/bucket1/obj1')
         self.assertEqual(req.environ['PATH_INFO'], '/obj1')
 
+        req = Request.blank('/obj2', environ=environ, headers=headers)
+        sigv2_req = S3Request(
+            req.environ, conf=Config({
+                'storage_domains': ['alternate.domain', 's3.test.com']}))
+        uri = sigv2_req._canonical_uri()
+        self.assertEqual(uri, '/bucket1/obj2')
+        self.assertEqual(req.environ['PATH_INFO'], '/obj2')
+
+        # Now check the other storage_domain
+        environ = {
+            'HTTP_HOST': 'bucket1.alternate.domain',
+            'REQUEST_METHOD': 'GET'}
+        req = Request.blank('/obj2', environ=environ, headers=headers)
+        sigv2_req = S3Request(
+            req.environ, conf=Config({
+                'storage_domains': ['alternate.domain', 's3.test.com']}))
+        uri = sigv2_req._canonical_uri()
+        self.assertEqual(uri, '/bucket1/obj2')
+        self.assertEqual(req.environ['PATH_INFO'], '/obj2')
+
+        # Non existent storage_domain means we can't find the  container
+        environ = {
+            'HTTP_HOST': 'bucket1.incorrect.domain',
+            'REQUEST_METHOD': 'GET'}
+        req = Request.blank('/obj2', environ=environ, headers=headers)
+        sigv2_req = S3Request(
+            req.environ, conf=Config({
+                'storage_domains': ['alternate.domain', 's3.test.com']}))
+        uri = sigv2_req._canonical_uri()
+        # uo oh, no bucket
+        self.assertEqual(uri, '/obj2')
+        self.assertEqual(sigv2_req.container_name, 'obj2')
+
         environ = {
             'HTTP_HOST': 's3.test.com',
             'REQUEST_METHOD': 'GET'}
@@ -701,7 +734,7 @@ class TestRequest(S3ApiTestCase):
             'X-Amz-Date': x_amz_date}
 
         # Virtual hosted-style
-        self.s3api.conf.storage_domain = 's3.test.com'
+        self.s3api.conf.storage_domains = ['s3.test.com']
         req = Request.blank('/', environ=environ, headers=headers)
         sigv4_req = SigV4Request(req.environ)
         uri = sigv4_req._canonical_uri()
@@ -721,7 +754,7 @@ class TestRequest(S3ApiTestCase):
             'REQUEST_METHOD': 'GET'}
 
         # Path-style
-        self.s3api.conf.storage_domain = ''
+        self.s3api.conf.storage_domains = []
         req = Request.blank('/', environ=environ, headers=headers)
         sigv4_req = SigV4Request(req.environ)
         uri = sigv4_req._canonical_uri()
@@ -749,7 +782,7 @@ class TestRequest(S3ApiTestCase):
                               'bWq2s1WEIj+Ydj0vQ697zp+IXMU='),
         })
         sigv2_req = S3Request(req.environ, conf=Config({
-            'storage_domain': 's3.amazonaws.com'}))
+            'storage_domains': ['s3.amazonaws.com']}))
         expected_sts = b'\n'.join([
             b'GET',
             b'',
@@ -769,7 +802,7 @@ class TestRequest(S3ApiTestCase):
                               'MyyxeRY7whkBe+bq8fHCL/2kKUg='),
         })
         sigv2_req = S3Request(req.environ, conf=Config({
-            'storage_domain': 's3.amazonaws.com'}))
+            'storage_domains': ['s3.amazonaws.com']}))
         expected_sts = b'\n'.join([
             b'PUT',
             b'',
@@ -790,7 +823,7 @@ class TestRequest(S3ApiTestCase):
                                   'htDYFYduRNen8P9ZfE/s9SuKy0U='),
             })
         sigv2_req = S3Request(req.environ, conf=Config({
-            'storage_domain': 's3.amazonaws.com'}))
+            'storage_domains': ['s3.amazonaws.com']}))
         expected_sts = b'\n'.join([
             b'GET',
             b'',
@@ -819,7 +852,7 @@ class TestRequest(S3ApiTestCase):
                               'bWq2s1WEIj+Ydj0vQ697zp+IXMU='),
         })
         sigv2_req = S3Request(req.environ, Config({
-            'storage_domain': 's3.amazonaws.com'}))
+            'storage_domains': ['s3.amazonaws.com']}))
         # This is a failure case with utf-8 non-ascii multi-bytes charactor
         # but we expect to return just False instead of exceptions
         self.assertFalse(sigv2_req.check_signature(
@@ -837,7 +870,7 @@ class TestRequest(S3ApiTestCase):
             'X-Amz-Date': amz_date_header
         })
         sigv4_req = SigV4Request(
-            req.environ, Config({'storage_domain': 's3.amazonaws.com'}))
+            req.environ, Config({'storage_domains': ['s3.amazonaws.com']}))
         self.assertFalse(sigv4_req.check_signature(
             u'\u30c9\u30e9\u30b4\u30f3'))
 
@@ -858,7 +891,7 @@ class TestRequest(S3ApiTestCase):
             'X-Amz-Date': '20210104T102623Z'}
 
         # Virtual hosted-style
-        self.s3api.conf.storage_domain = 's3.test.com'
+        self.s3api.conf.storage_domains = ['s3.test.com']
         req = Request.blank('/', environ=environ, headers=headers)
         sigv4_req = SigV4Request(req.environ)
         self.assertTrue(
@@ -868,7 +901,7 @@ class TestRequest(S3ApiTestCase):
     @patch.object(S3Request, '_validate_dates', lambda *a: None)
     def test_check_sigv4_req_zero_content_length_sha256(self):
         # Virtual hosted-style
-        self.s3api.conf.storage_domain = 's3.test.com'
+        self.s3api.conf.storage_domains = ['s3.test.com']
 
         # bad sha256
         environ = {

test/unit/common/middleware/s3api/test_utils.py

@@ -133,7 +133,7 @@ class TestS3ApiUtils(unittest.TestCase):
 class TestConfig(unittest.TestCase):
 
     def _assert_defaults(self, conf):
-        self.assertEqual('', conf.storage_domain)
+        self.assertEqual([], conf.storage_domains)
         self.assertEqual('us-east-1', conf.location)
         self.assertFalse(conf.force_swift_request_proxy_log)
         self.assertTrue(conf.dns_compliant_bucket_names)
@@ -146,7 +146,7 @@ class TestConfig(unittest.TestCase):
         # deliberately brittle so new defaults will need to be added to test
         conf = utils.Config()
         self._assert_defaults(conf)
-        del conf.storage_domain
+        del conf.storage_domains
         del conf.location
         del conf.force_swift_request_proxy_log
         del conf.dns_compliant_bucket_names