From e413c650e385ba44523e99e023ada0a44aee8c7c Mon Sep 17 00:00:00 2001 From: Romain LE DISEZ Date: Thu, 9 Jan 2020 09:08:25 -0500 Subject: [PATCH] Allow bulk to fwd some headers at tar extraction Whitelisted headers include X-Delete-At/X-Delete-After and all Object metadata headers (X-Object-Meta-*) Closes-Bug: 1857546 Change-Id: If5fb164693e395f89d57899fb8ab355f1e3f817c (cherry picked from commit ff0753fe19be3d7442aef3972e08adc48d25fe93) --- swift/common/middleware/bulk.py | 11 +++++++ test/unit/common/middleware/test_bulk.py | 39 +++++++++++++++++++++--- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/swift/common/middleware/bulk.py b/swift/common/middleware/bulk.py index 9c4f613839..0ca2535da6 100644 --- a/swift/common/middleware/bulk.py +++ b/swift/common/middleware/bulk.py @@ -98,6 +98,10 @@ The bulk middleware will handle xattrs stored by both GNU and BSD tar (2). Only xattrs ``user.mime_type`` and ``user.meta.*`` are processed. Other attributes are ignored. +In addition to the extended attributes, the object metadata and the +x-delete-at/x-delete-after headers set in the request are also assigned to the +extracted objects. + Notes: (1) The POSIX 1003.1-2001 (pax) format. The default format on GNU tar @@ -206,6 +210,7 @@ from swift.common.utils import get_logger, register_swift_info, \ StreamingPile from swift.common import constraints from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT +from swift.common.request_helpers import is_user_meta from swift.common.wsgi import make_subrequest @@ -622,6 +627,12 @@ class Bulk(object): 'X-Auth-Token': req.headers.get('X-Auth-Token'), } + # Copy some whitelisted headers to the subrequest + for k, v in req.headers.items(): + if ((k.lower() in ('x-delete-at', 'x-delete-after')) + or is_user_meta('object', k)): + create_headers[k] = v + create_obj_req = make_subrequest( req.environ, method='PUT', path=wsgi_quote(destination), diff --git a/test/unit/common/middleware/test_bulk.py b/test/unit/common/middleware/test_bulk.py index 93c664b3b3..5de8b85dc9 100644 --- a/test/unit/common/middleware/test_bulk.py +++ b/test/unit/common/middleware/test_bulk.py @@ -41,12 +41,21 @@ class FakeApp(object): def __init__(self): self.calls = 0 self.delete_paths = [] + self.put_paths = [] self.max_pathlen = 100 self.del_cont_total_calls = 2 self.del_cont_cur_call = 0 def __call__(self, env, start_response): self.calls += 1 + if env.get('swift.source') in ('EA', 'BD'): + assert not env.get('swift.proxy_access_log_made') + if not six.PY2: + # Check that it's valid WSGI + assert all(0 <= ord(c) <= 255 for c in env['PATH_INFO']) + + if env['REQUEST_METHOD'] == 'PUT': + self.put_paths.append(env['PATH_INFO']) if env['PATH_INFO'].startswith('/unauth/'): if env['PATH_INFO'].endswith('/c/f_ok'): return Response(status='204 No Content')(env, start_response) @@ -224,8 +233,16 @@ class TestUntarMetadata(unittest.TestCase): req = Request.blank('/v1/a/c?extract-archive=tar') req.environ['REQUEST_METHOD'] = 'PUT' req.environ['wsgi.input'] = tar_ball - req.headers['transfer-encoding'] = 'chunked' - req.headers['accept'] = 'application/json;q=1.0' + # Since there should be a proxy-logging left of us... + req.environ['swift.proxy_access_log_made'] = True + req.headers.update({ + 'transfer-encoding': 'chunked', + 'accept': 'application/json;q=1.0', + 'X-Delete-At': '1577383915', + 'X-Object-Meta-Dog': 'Rantanplan', + 'X-Horse': 'Jolly Jumper', + 'X-Object-Meta-Cat': 'tabby', + }) resp = req.get_response(self.bulk) self.assertEqual(resp.status_int, 200) @@ -244,12 +261,19 @@ class TestUntarMetadata(unittest.TestCase): self.assertEqual( put1_headers.get('X-Object-Meta-Afternoon-Snack'), 'gigantic bucket of coffee') + self.assertEqual(put1_headers.get('X-Delete-At'), '1577383915') + self.assertEqual(put1_headers.get('X-Object-Meta-Dog'), 'Rantanplan') + self.assertEqual(put1_headers.get('X-Object-Meta-Cat'), 'tabby') + self.assertIsNone(put1_headers.get('X-Horse')) put2_headers = HeaderKeyDict(self.app.calls_with_headers[2][2]) self.assertEqual(put2_headers.get('X-Object-Meta-Muppet'), 'bert') self.assertEqual(put2_headers.get('X-Object-Meta-Cat'), 'fluffy') self.assertIsNone(put2_headers.get('Content-Type')) self.assertIsNone(put2_headers.get('X-Object-Meta-Blah')) + self.assertEqual(put2_headers.get('X-Delete-At'), '1577383915') + self.assertEqual(put2_headers.get('X-Object-Meta-Dog'), 'Rantanplan') + self.assertIsNone(put2_headers.get('X-Horse')) class TestUntar(unittest.TestCase): @@ -610,6 +634,7 @@ class TestUntar(unittest.TestCase): def test_extract_tar_fail_unicode(self): dir_tree = [{'sub_dir1': ['sub1_file1']}, {'sub_dir2': [b'sub2\xdefile1', 'sub2_file2']}, + {b'good_\xe2\x98\x83': [{'still_good': b'\xe2\x98\x83'}]}, {b'sub_\xdedir3': [{'sub4_dir1': 'sub4_file1'}]}] self.build_tar(dir_tree) req = Request.blank('/tar_works/acc/', @@ -619,13 +644,18 @@ class TestUntar(unittest.TestCase): req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, '') resp_data = utils.json.loads(resp_body) - self.assertEqual(self.app.calls, 4) - self.assertEqual(resp_data['Number Files Created'], 2) + self.assertEqual(self.app.calls, 6) + self.assertEqual(resp_data['Number Files Created'], 3) self.assertEqual(resp_data['Response Status'], '400 Bad Request') self.assertEqual( resp_data['Errors'], [['sub_dir2/sub2%DEfile1', '412 Precondition Failed'], ['sub_%DEdir3/sub4_dir1/sub4_file1', '412 Precondition Failed']]) + self.assertEqual(self.app.put_paths, [ + '/tar_works/acc/sub_dir1/sub1_file1', + '/tar_works/acc/sub_dir2/sub2_file2', + '/tar_works/acc/good_\xe2\x98\x83/still_good/\xe2\x98\x83', + ]) def test_get_response_body(self): txt_body = bulk.get_response_body( @@ -1027,5 +1057,6 @@ class TestSwiftInfo(unittest.TestCase): swift_info['bulk_delete'].get('max_failed_deletes'), numbers.Integral)) + if __name__ == '__main__': unittest.main()