Merge "Add support for more characters in header keys"

swift3/request.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import base64
+from collections import defaultdict
 from email.header import Header
 from hashlib import sha1, sha256, md5
 import hmac
@@ -266,9 +267,18 @@ class SigV4Mixin(object):
 
         :return : dict of headers to sign, the keys are all lower case
         """
-        headers_lower_dict = dict(
+        if 'headers_raw' in self.environ:  # eventlet >= 0.19.0
-            (k.lower().strip(), ' '.join(_header_strip(v or '').split()))
+            # See https://github.com/eventlet/eventlet/commit/67ec999
-            for (k, v) in six.iteritems(self.headers))
+            headers_lower_dict = defaultdict(list)
+            for key, value in self.environ['headers_raw']:
+                headers_lower_dict[key.lower().strip()].append(
+                    ' '.join(_header_strip(value or '').split()))
+            headers_lower_dict = {k: ','.join(v)
+                                  for k, v in headers_lower_dict.items()}
+        else:  # mostly-functional fallback
+            headers_lower_dict = dict(
+                (k.lower().strip(), ' '.join(_header_strip(v or '').split()))
+                for (k, v) in six.iteritems(self.headers))
 
         if 'host' in headers_lower_dict and re.match(
                 'Boto/2.[0-9].[0-2]',
@@ -280,7 +290,7 @@ class SigV4Mixin(object):
                 headers_lower_dict['host'].split(':')[0]
 
         headers_to_sign = [
-            (key, value) for key, value in headers_lower_dict.items()
+            (key, value) for key, value in sorted(headers_lower_dict.items())
             if key in self._signed_headers]
 
         if len(headers_to_sign) != len(self._signed_headers):
@@ -291,7 +301,7 @@ class SigV4Mixin(object):
             # process.
             raise SignatureDoesNotMatch()
 
-        return dict(headers_to_sign)
+        return headers_to_sign
 
     def _canonical_uri(self):
         """
@@ -329,13 +339,12 @@ class SigV4Mixin(object):
         # host:iam.amazonaws.com
         # x-amz-date:20150830T123600Z
         headers_to_sign = self._headers_to_sign()
-        cr.append('\n'.join(
+        cr.append(''.join('%s:%s\n' % (key, value)
-            ['%s:%s' % (key, value) for key, value in
+                          for key, value in headers_to_sign))
-             sorted(headers_to_sign.items())]) + '\n')
 
         # 5. Add signed headers into canonical request like
         # content-type;host;x-amz-date
-        cr.append(';'.join(sorted(headers_to_sign)))
+        cr.append(';'.join(k for k, v in headers_to_sign))
 
         # 6. Add payload string at the tail
         if 'X-Amz-Credential' in self.params:
@@ -800,9 +809,20 @@ class Request(swob.Request):
                _header_strip(self.headers.get('Content-MD5')) or '',
                _header_strip(self.headers.get('Content-Type')) or '']
 
-        for amz_header in sorted((key.lower() for key in self.headers
+        if 'headers_raw' in self.environ:  # eventlet >= 0.19.0
-                                  if key.lower().startswith('x-amz-'))):
+            # See https://github.com/eventlet/eventlet/commit/67ec999
-            amz_headers[amz_header] = self.headers[amz_header]
+            amz_headers = defaultdict(list)
+            for key, value in self.environ['headers_raw']:
+                key = key.lower()
+                if not key.startswith('x-amz-'):
+                    continue
+                amz_headers[key.strip()].append(value.strip())
+            amz_headers = dict((key, ','.join(value))
+                               for key, value in amz_headers.items())
+        else:  # mostly-functional fallback
+            amz_headers = dict((key.lower(), value)
+                               for key, value in self.headers.items()
+                               if key.lower().startswith('x-amz-'))
 
         if self._is_header_auth:
             if 'x-amz-date' in amz_headers:
@@ -816,8 +836,8 @@ class Request(swob.Request):
             # but as a sanity check...
             raise AccessDenied()
 
-        for k in sorted(key.lower() for key in amz_headers):
+        for key, value in sorted(amz_headers.items()):
-            buf.append("%s:%s" % (k, amz_headers[k]))
+            buf.append("%s:%s" % (key, value))
 
         path = self._canonical_uri()
         if self.query_string:
@@ -903,15 +923,48 @@ class Request(swob.Request):
 
         env = self.environ.copy()
 
-        for key in self.environ:
+        def sanitize(value):
-            if key.startswith('HTTP_X_AMZ_META_'):
+            if set(value).issubset(string.printable):
-                if not(set(env[key]).issubset(string.printable)):
+                return value
-                    env[key] = Header(env[key], 'UTF-8').encode()
+
-                    if env[key].startswith('=?utf-8?q?'):
+            value = Header(value, 'UTF-8').encode()
-                        env[key] = '=?UTF-8?Q?' + env[key][10:]
+            if value.startswith('=?utf-8?q?'):
-                    elif env[key].startswith('=?utf-8?b?'):
+                return '=?UTF-8?Q?' + value[10:]
-                        env[key] = '=?UTF-8?B?' + env[key][10:]
+            elif value.startswith('=?utf-8?b?'):
-                env['HTTP_X_OBJECT_META_' + key[16:]] = env[key]
+                return '=?UTF-8?B?' + value[10:]
+            else:
+                return value
+
+        if 'headers_raw' in env:  # eventlet >= 0.19.0
+            # See https://github.com/eventlet/eventlet/commit/67ec999
+            for key, value in env['headers_raw']:
+                if not key.lower().startswith('x-amz-meta-'):
+                    continue
+                # AWS ignores user-defined headers with these characters
+                if any(c in key for c in ' "),/;<=>?@[\\]{}'):
+                    # NB: apparently, '(' *is* allowed
+                    continue
+                # Note that this may have already been deleted, e.g. if the
+                # client sent multiple headers with the same name, or both
+                # x-amz-meta-foo-bar and x-amz-meta-foo_bar
+                env.pop('HTTP_' + key.replace('-', '_').upper(), None)
+                # Need to preserve underscores. Since we know '=' can't be
+                # present, quoted-printable seems appropriate.
+                key = key.replace('_', '=5F').replace('-', '_').upper()
+                key = 'HTTP_X_OBJECT_META_' + key[11:]
+                if key in env:
+                    env[key] += ',' + sanitize(value)
+                else:
+                    env[key] = sanitize(value)
+        else:  # mostly-functional fallback
+            for key in self.environ:
+                if not key.startswith('HTTP_X_AMZ_META_'):
+                    continue
+                # AWS ignores user-defined headers with these characters
+                if any(c in key for c in ' "),/;<=>?@[\\]{}'):
+                    # NB: apparently, '(' *is* allowed
+                    continue
+                env['HTTP_X_OBJECT_META_' + key[16:]] = sanitize(env[key])
                 del env[key]
 
         if 'HTTP_X_AMZ_COPY_SOURCE' in env:

swift3/response.py

@@ -100,7 +100,10 @@ class Response(ResponseBase, swob.Response):
             _key = key.lower()
 
             if _key.startswith('x-object-meta-'):
-                headers['x-amz-meta-' + _key[14:]] = val
+                # Note that AWS ignores user-defined headers with '=' in the
+                # header name. We translated underscores to '=5F' on the way
+                # in, though.
+                headers['x-amz-meta-' + _key[14:].replace('=5f', '_')] = val
             elif _key in ('content-length', 'content-type',
                           'content-range', 'content-encoding',
                           'content-disposition', 'content-language',

swift3/test/functional/test_object.py

@@ -318,7 +318,9 @@ class TestSwift3Object(Swift3FunctionalTestCase):
         self.assertCommonResponseHeaders(headers)
         self._assertObjectEtag(self.bucket, obj, etag)
 
-    def _test_put_object_headers(self, req_headers):
+    def _test_put_object_headers(self, req_headers, expected_headers=None):
+        if expected_headers is None:
+            expected_headers = req_headers
         obj = 'object'
         content = 'abcdefghij'
         etag = md5(content).hexdigest()
@@ -328,7 +330,7 @@ class TestSwift3Object(Swift3FunctionalTestCase):
         self.assertEqual(status, 200)
         status, headers, body = \
             self.conn.make_request('HEAD', self.bucket, obj)
-        for header, value in req_headers.items():
+        for header, value in expected_headers.items():
             self.assertIn(header.lower(), headers)
             self.assertEqual(headers[header.lower()], value)
         self.assertCommonResponseHeaders(headers)
@@ -339,6 +341,21 @@ class TestSwift3Object(Swift3FunctionalTestCase):
             'X-Amz-Meta-Bar': 'foo',
             'X-Amz-Meta-Bar2': 'foo2'})
 
+    def test_put_object_weird_metadata(self):
+        req_headers = dict(
+            ('x-amz-meta-' + c, c)
+            for c in '!"#$%&\'()*+-./<=>?@[\\]^`{|}~')
+        exp_headers = dict(
+            ('x-amz-meta-' + c, c)
+            for c in '!#$%&\'(*+-.^`|~')
+        self._test_put_object_headers(req_headers, exp_headers)
+
+    def test_put_object_underscore_in_metadata(self):
+        # Break this out separately for ease of testing pre-0.19.0 eventlet
+        self._test_put_object_headers({
+            'X-Amz-Meta-Foo-Bar': 'baz',
+            'X-Amz-Meta-Foo_Bar': 'also baz'})
+
     def test_put_object_content_headers(self):
         self._test_put_object_headers({
             'Content-Type': 'foo/bar',

swift3/test/unit/test_request.py

@@ -391,8 +391,8 @@ class TestRequest(Swift3TestCase):
             'Authorization':
                 'AWS4-HMAC-SHA256 '
                 'Credential=test/20130524/US/s3/aws4_request, '
-                'SignedHeaders=host;%s,'
+                'SignedHeaders=%s,'
-                'Signature=X' % included_header,
+                'Signature=X' % ';'.join(sorted(['host', included_header])),
             'X-Amz-Content-SHA256': '0123456789'}
 
         headers.update(date_header)
@@ -551,11 +551,10 @@ class TestRequest(Swift3TestCase):
         sigv4_req = SigV4Request(req.environ)
 
         headers_to_sign = sigv4_req._headers_to_sign()
-        self.assertEqual(['host', 'x-amz-content-sha256', 'x-amz-date'],
+        self.assertEqual(headers_to_sign, [
-                         sorted(headers_to_sign.keys()))
+            ('host', 'localhost:80'),
-        self.assertEqual(headers_to_sign['host'], 'localhost:80')
+            ('x-amz-content-sha256', '0123456789'),
-        self.assertEqual(headers_to_sign['x-amz-date'], x_amz_date)
+            ('x-amz-date', x_amz_date)])
-        self.assertEqual(headers_to_sign['x-amz-content-sha256'], '0123456789')
 
         # no x-amz-date
         headers = {
@@ -571,10 +570,9 @@ class TestRequest(Swift3TestCase):
         sigv4_req = SigV4Request(req.environ)
 
         headers_to_sign = sigv4_req._headers_to_sign()
-        self.assertEqual(['host', 'x-amz-content-sha256'],
+        self.assertEqual(headers_to_sign, [
-                         sorted(headers_to_sign.keys()))
+            ('host', 'localhost:80'),
-        self.assertEqual(headers_to_sign['host'], 'localhost:80')
+            ('x-amz-content-sha256', '0123456789')])
-        self.assertEqual(headers_to_sign['x-amz-content-sha256'], '0123456789')
 
         # SignedHeaders says, host and x-amz-date included but there is not
         # X-Amz-Date header