From 284bbdd391b24823f1f1154577b2b68575e7f7e9 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 11 Oct 2018 23:25:32 +0000 Subject: [PATCH] Add slo_manifest_hook callback ... to allow other middlewares to impose additional constraints on or make edits to SLO manifests before being written. The callback takes a single argument: the python list that represents the manifest to be written. All the normal list operations listed at https://docs.python.org/2/library/stdtypes.html#mutable-sequence-types are available to make changes to that before SLO serializes it as JSON. The callback may return a list of problematic segments; each item in the list should be a tuple of (quoted object name, description of problem) This will be useful both for s3api minimum segment size validation and creating tar large objects. Change-Id: I198c5196e0221a72b14597a06e5ce3c4b2bbf436 Related-Bug: #1636663 --- swift/common/middleware/slo.py | 9 +++++ test/unit/common/middleware/test_slo.py | 53 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index 39f18d593c..7d15aab1fb 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -1227,6 +1227,15 @@ class StaticLargeObject(object): data_for_storage[i] = seg_data total_size += segment_length + # Middleware left of SLO can add a callback to the WSGI + # environment to perform additional validation and/or + # manipulation on the manifest that will be written. + hook = req.environ.get('swift.callback.slo_manifest_hook') + if hook: + more_problems = hook(data_for_storage) + if more_problems: + problem_segments.extend(more_problems) + if problem_segments: err = HTTPBadRequest(content_type=out_content_type) resp_dict = {} diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 9b08a45c9e..7da774f96b 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -964,6 +964,59 @@ class TestSloPutManifest(SloTestCase): self.assertEqual('a', manifest_data[0]['hash']) self.assertEqual('b', manifest_data[1]['hash']) + def test_handle_multipart_put_with_manipulator_callback(self): + def data_inserter(manifest): + for i in range(len(manifest), -1, -1): + manifest.insert(i, {'data': 'WA=='}) + + good_data = json.dumps([ + {'path': '/checktest/a_1'}, + {'path': '/checktest/b_2'}]) + req = Request.blank( + '/v1/AUTH_test/checktest/man_3?multipart-manifest=put', + environ={'REQUEST_METHOD': 'PUT', + 'swift.callback.slo_manifest_hook': data_inserter}, + body=good_data) + status, headers, body = self.call_slo(req) + self.assertEqual(self.app.call_count, 3) + + # Check that we still populated the manifest properly from our HEADs + req = Request.blank( + '/v1/AUTH_test/checktest/man_3?multipart-manifest=put', + environ={'REQUEST_METHOD': 'GET'}) + status, headers, body = self.call_app(req) + manifest_data = json.loads(body) + self.assertEqual([ + {k: v for k, v in item.items() + if k in ('name', 'bytes', 'hash', 'data')} + for item in manifest_data + ], [ + {'data': 'WA=='}, + {'name': '/checktest/a_1', 'bytes': 1, 'hash': 'a'}, + {'data': 'WA=='}, + {'name': '/checktest/b_2', 'bytes': 2, 'hash': 'b'}, + {'data': 'WA=='}, + ]) + + def test_handle_multipart_put_with_validator_callback(self): + def complainer(manifest): + return [(item['name'], "Don't wanna") for item in manifest] + + good_data = json.dumps([ + {'path': '/checktest/a_1'}, + {'path': '/checktest/b_2'}]) + req = Request.blank( + '/v1/AUTH_test/checktest/man_3?multipart-manifest=put', + environ={'REQUEST_METHOD': 'PUT', + 'swift.callback.slo_manifest_hook': complainer}, + body=good_data) + status, headers, body = self.call_slo(req) + self.assertEqual(self.app.call_count, 2) + self.assertEqual(status, '400 Bad Request') + body = body.split('\n') + self.assertIn("/checktest/a_1, Don't wanna", body) + self.assertIn("/checktest/b_2, Don't wanna", body) + def test_handle_unsatisfiable_ranges(self): bad_data = json.dumps( [{'path': '/checktest/a_1', 'etag': None,