Browse Source

Verify client input for v4 signatures

Previously, we would use the X-Amz-Content-SHA256 value when calculating
signatures, but wouldn't actually check the content that was sent. This
would allow a malicious third party that managed to capture the headers
for an object upload to overwrite that with arbitrary content provided
they could do so within the 5-minute clock-skew window.

Now, we wrap the wsgi.input that's sent on to the proxy-server app to
hash content as it's read and raise an error if there's a mismatch. Note
that clients using presigned-urls to upload have no defense against a
similar replay attack.

Notwithstanding the above security consideration, this *also* provides
better assurances that the client's payload was received correctly. Note
that this *does not* attempt to send an etag in footers, however, so the
proxy-to-object-server connection is not guarded against bit-flips.

In the future, Swift will hopefully grow a way to perform SHA256
verification on the object-server. This would offer two main benefits:

  - End-to-end message integrity checking.
  - Move CPU load of calculating the hash from the proxy (which is
    somewhat CPU-bound) to the object-server (which tends to have CPU to
    spare).

Change-Id: I61eb12455c37376be4d739eee55a5f439216f0e9
Closes-Bug: 1765834
tags/2.21.0
Tim Burke 6 months ago
parent
commit
3a8f5dbf9c

+ 53
- 7
swift/common/middleware/s3api/s3request.py View File

@@ -24,7 +24,8 @@ import six
24 24
 from six.moves.urllib.parse import quote, unquote, parse_qsl
25 25
 import string
26 26
 
27
-from swift.common.utils import split_path, json, get_swift_info
27
+from swift.common.utils import split_path, json, get_swift_info, \
28
+    close_if_possible
28 29
 from swift.common import swob
29 30
 from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
30 31
     HTTP_NO_CONTENT, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, \
@@ -110,6 +111,34 @@ def _header_acl_property(resource):
110 111
                     doc='Get and set the %s acl property' % resource)
111 112
 
112 113
 
114
+class HashingInput(object):
115
+    """
116
+    wsgi.input wrapper to verify the hash of the input as it's read.
117
+    """
118
+    def __init__(self, reader, content_length, hasher, expected_hex_hash):
119
+        self._input = reader
120
+        self._to_read = content_length
121
+        self._hasher = hasher()
122
+        self._expected = expected_hex_hash
123
+
124
+    def read(self, size=None):
125
+        chunk = self._input.read(size)
126
+        self._hasher.update(chunk)
127
+        self._to_read -= len(chunk)
128
+        if self._to_read < 0 or (size > len(chunk) and self._to_read) or (
129
+                self._to_read == 0 and
130
+                self._hasher.hexdigest() != self._expected):
131
+            self.close()
132
+            # Since we don't return the last chunk, the PUT never completes
133
+            raise swob.HTTPUnprocessableEntity(
134
+                'The X-Amz-Content-SHA56 you specified did not match '
135
+                'what we received.')
136
+        return chunk
137
+
138
+    def close(self):
139
+        close_if_possible(self._input)
140
+
141
+
113 142
 class SigV4Mixin(object):
114 143
     """
115 144
     A request class mixin to provide S3 signature v4 functionality
@@ -401,6 +430,20 @@ class SigV4Mixin(object):
401 430
             raise InvalidRequest(msg)
402 431
         else:
403 432
             hashed_payload = self.headers['X-Amz-Content-SHA256']
433
+            if self.content_length == 0:
434
+                if hashed_payload != sha256().hexdigest():
435
+                    raise BadDigest(
436
+                        'The X-Amz-Content-SHA56 you specified did not match '
437
+                        'what we received.')
438
+            elif self.content_length:
439
+                self.environ['wsgi.input'] = HashingInput(
440
+                    self.environ['wsgi.input'],
441
+                    self.content_length,
442
+                    sha256,
443
+                    hashed_payload)
444
+            # else, not provided -- Swift will kick out a 411 Length Required
445
+            # which will get translated back to a S3-style response in
446
+            # S3Request._swift_error_codes
404 447
         cr.append(hashed_payload)
405 448
         return '\n'.join(cr).encode('utf-8')
406 449
 
@@ -1264,12 +1307,15 @@ class S3Request(swob.Request):
1264 1307
         sw_req = self.to_swift_req(method, container, obj, headers=headers,
1265 1308
                                    body=body, query=query)
1266 1309
 
1267
-        sw_resp = sw_req.get_response(app)
1268
-
1269
-        # reuse account and tokens
1270
-        _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
1271
-                                        2, 3, True)
1272
-        self.account = utf8encode(self.account)
1310
+        try:
1311
+            sw_resp = sw_req.get_response(app)
1312
+        except swob.HTTPException as err:
1313
+            sw_resp = err
1314
+        else:
1315
+            # reuse account and tokens
1316
+            _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
1317
+                                            2, 3, True)
1318
+            self.account = utf8encode(self.account)
1273 1319
 
1274 1320
         resp = S3Response.from_swift_resp(sw_resp)
1275 1321
         status = resp.status_int  # pylint: disable-msg=E1101

+ 54
- 0
test/unit/common/middleware/s3api/test_obj.py View File

@@ -469,6 +469,60 @@ class TestS3ApiObj(S3ApiTestCase):
469 469
         # Check that s3api converts a Content-MD5 header into an etag.
470 470
         self.assertEqual(headers['etag'], etag)
471 471
 
472
+    @s3acl
473
+    def test_object_PUT_v4(self):
474
+        body_sha = hashlib.sha256(self.object_body).hexdigest()
475
+        req = Request.blank(
476
+            '/bucket/object',
477
+            environ={'REQUEST_METHOD': 'PUT'},
478
+            headers={
479
+                'Authorization':
480
+                    'AWS4-HMAC-SHA256 '
481
+                    'Credential=test:tester/%s/us-east-1/s3/aws4_request, '
482
+                    'SignedHeaders=host;x-amz-date, '
483
+                    'Signature=hmac' % (
484
+                        self.get_v4_amz_date_header().split('T', 1)[0]),
485
+                'x-amz-date': self.get_v4_amz_date_header(),
486
+                'x-amz-storage-class': 'STANDARD',
487
+                'x-amz-content-sha256': body_sha,
488
+                'Date': self.get_date_header()},
489
+            body=self.object_body)
490
+        req.date = datetime.now()
491
+        req.content_type = 'text/plain'
492
+        status, headers, body = self.call_s3api(req)
493
+        self.assertEqual(status.split()[0], '200')
494
+        # Check that s3api returns an etag header.
495
+        self.assertEqual(headers['etag'],
496
+                         '"%s"' % self.response_headers['etag'])
497
+
498
+        _, _, headers = self.swift.calls_with_headers[-1]
499
+        # No way to determine ETag to send
500
+        self.assertNotIn('etag', headers)
501
+
502
+    @s3acl
503
+    def test_object_PUT_v4_bad_hash(self):
504
+        req = Request.blank(
505
+            '/bucket/object',
506
+            environ={'REQUEST_METHOD': 'PUT'},
507
+            headers={
508
+                'Authorization':
509
+                    'AWS4-HMAC-SHA256 '
510
+                    'Credential=test:tester/%s/us-east-1/s3/aws4_request, '
511
+                    'SignedHeaders=host;x-amz-date, '
512
+                    'Signature=hmac' % (
513
+                        self.get_v4_amz_date_header().split('T', 1)[0]),
514
+                'x-amz-date': self.get_v4_amz_date_header(),
515
+                'x-amz-storage-class': 'STANDARD',
516
+                'x-amz-content-sha256': 'not the hash',
517
+                'Date': self.get_date_header()},
518
+            body=self.object_body)
519
+        req.date = datetime.now()
520
+        req.content_type = 'text/plain'
521
+        status, headers, body = self.call_s3api(req)
522
+        self.assertEqual(status.split()[0], '400')
523
+        print(body)
524
+        self.assertEqual(self._get_error_code(body), 'BadDigest')
525
+
472 526
     def test_object_PUT_headers(self):
473 527
         content_md5 = self.etag.decode('hex').encode('base64').strip()
474 528
 

+ 75
- 1
test/unit/common/middleware/s3api/test_s3request.py View File

@@ -13,9 +13,12 @@
13 13
 # See the License for the specific language governing permissions and
14 14
 # limitations under the License.
15 15
 
16
+import hashlib
16 17
 from mock import patch, MagicMock
17 18
 import unittest
18 19
 
20
+from six import BytesIO
21
+
19 22
 from swift.common import swob
20 23
 from swift.common.swob import Request, HTTPNoContent
21 24
 from swift.common.middleware.s3api.utils import mktime
@@ -24,7 +27,7 @@ from swift.common.middleware.s3api.subresource import ACL, User, Owner, \
24 27
     Grant, encode_acl
25 28
 from test.unit.common.middleware.s3api.test_s3api import S3ApiTestCase
26 29
 from swift.common.middleware.s3api.s3request import S3Request, \
27
-    S3AclRequest, SigV4Request, SIGV4_X_AMZ_DATE_FORMAT
30
+    S3AclRequest, SigV4Request, SIGV4_X_AMZ_DATE_FORMAT, HashingInput
28 31
 from swift.common.middleware.s3api.s3response import InvalidArgument, \
29 32
     NoSuchBucket, InternalError, \
30 33
     AccessDenied, SignatureDoesNotMatch, RequestTimeTooSkewed
@@ -802,5 +805,76 @@ class TestRequest(S3ApiTestCase):
802 805
             u'\u30c9\u30e9\u30b4\u30f3'))
803 806
 
804 807
 
808
+class TestHashingInput(S3ApiTestCase):
809
+    def test_good(self):
810
+        raw = b'123456789'
811
+        wrapped = HashingInput(BytesIO(raw), 9, hashlib.md5,
812
+                               hashlib.md5(raw).hexdigest())
813
+        self.assertEqual(b'1234', wrapped.read(4))
814
+        self.assertEqual(b'56', wrapped.read(2))
815
+        # trying to read past the end gets us whatever's left
816
+        self.assertEqual(b'789', wrapped.read(4))
817
+        # can continue trying to read -- but it'll be empty
818
+        self.assertEqual(b'', wrapped.read(2))
819
+
820
+        self.assertFalse(wrapped._input.closed)
821
+        wrapped.close()
822
+        self.assertTrue(wrapped._input.closed)
823
+
824
+    def test_empty(self):
825
+        wrapped = HashingInput(BytesIO(b''), 0, hashlib.sha256,
826
+                               hashlib.sha256(b'').hexdigest())
827
+        self.assertEqual(b'', wrapped.read(4))
828
+        self.assertEqual(b'', wrapped.read(2))
829
+
830
+        self.assertFalse(wrapped._input.closed)
831
+        wrapped.close()
832
+        self.assertTrue(wrapped._input.closed)
833
+
834
+    def test_too_long(self):
835
+        raw = b'123456789'
836
+        wrapped = HashingInput(BytesIO(raw), 8, hashlib.md5,
837
+                               hashlib.md5(raw).hexdigest())
838
+        self.assertEqual(b'1234', wrapped.read(4))
839
+        self.assertEqual(b'56', wrapped.read(2))
840
+        # even though the hash matches, there was more data than we expected
841
+        with self.assertRaises(swob.Response) as raised:
842
+            wrapped.read(3)
843
+        self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
844
+        # the error causes us to close the input
845
+        self.assertTrue(wrapped._input.closed)
846
+
847
+    def test_too_short(self):
848
+        raw = b'123456789'
849
+        wrapped = HashingInput(BytesIO(raw), 10, hashlib.md5,
850
+                               hashlib.md5(raw).hexdigest())
851
+        self.assertEqual(b'1234', wrapped.read(4))
852
+        self.assertEqual(b'56', wrapped.read(2))
853
+        # even though the hash matches, there was more data than we expected
854
+        with self.assertRaises(swob.Response) as raised:
855
+            wrapped.read(4)
856
+        self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
857
+        self.assertTrue(wrapped._input.closed)
858
+
859
+    def test_bad_hash(self):
860
+        raw = b'123456789'
861
+        wrapped = HashingInput(BytesIO(raw), 9, hashlib.sha256,
862
+                               hashlib.md5(raw).hexdigest())
863
+        self.assertEqual(b'1234', wrapped.read(4))
864
+        self.assertEqual(b'5678', wrapped.read(4))
865
+        with self.assertRaises(swob.Response) as raised:
866
+            wrapped.read(4)
867
+        self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
868
+        self.assertTrue(wrapped._input.closed)
869
+
870
+    def test_empty_bad_hash(self):
871
+        wrapped = HashingInput(BytesIO(b''), 0, hashlib.sha256, 'nope')
872
+        with self.assertRaises(swob.Response) as raised:
873
+            wrapped.read(3)
874
+        self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
875
+        # the error causes us to close the input
876
+        self.assertTrue(wrapped._input.closed)
877
+
878
+
805 879
 if __name__ == '__main__':
806 880
     unittest.main()

Loading…
Cancel
Save