From 37f23b072ee7a655ab9072f1d0bece0919dec61b Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Mon, 9 Oct 2017 17:31:30 -0700 Subject: [PATCH] Allow SLOs to have zero-byte last segments. Since we used to allow zero-byte last segments but now we don't, it can be difficult to deal with some old SLO manifests. Imagine you're writing some code to sync objects from Swift cluster A to Swift cluster B. You start off with just a GET from A piped into a PUT to B, and that works great until you hit a SLO manifest and B won't accept a 500GB object. So, you write some code to detect SLO manifests, sync their segments, then take the JSON manifest (?multipart-manifest=get) and sync *that* over. Now, life is good... until one day you get an exception notification that there's this manifest on cluster A that cluster B won't accept. Turns out that, back when Swift would take zero-byte final segments on SLOs (before commit 7f636a5), someone uploaded such a SLO to cluster A. Now, however, zero-byte final segments are invalid, so that SLO that exists over in cluster A can't just be copied to cluster B. A little coding later, your sync tool detects zero-byte final segments and removes them when copying a manifest. But now your ETags don't match between clusters, so you have to figure out some way to deal with that, and so you put it in metadata, but then you realize that your syncer might encounter a SLO which contains a sub-SLO which has a zero-byte final segment, and it's right about then that you start thinking about giving up on programming and getting a job as an elevator mechanic. This commit makes life easier for developers of such applications by allowing SLOs to have zero-byte segments again. Change-Id: Ia37880bbb435e269ec53b2963eb1b9121696d479 --- swift/common/middleware/slo.py | 9 +++++---- test/unit/common/middleware/test_slo.py | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index fdc2e9efc1..af2619daa4 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -366,7 +366,7 @@ def parse_and_validate_input(req_body, req_path): except (TypeError, ValueError): errors.append("Index %d: invalid size_bytes" % seg_index) continue - if seg_size < 1: + if seg_size < 1 and seg_index != (len(parsed_data) - 1): errors.append("Index %d: too small; each segment must be " "at least 1 byte." % (seg_index,)) @@ -948,7 +948,7 @@ class StaticLargeObject(object): agent='%(orig)s SLO MultipartPUT', swift_source='SLO') return obj_name, sub_req.get_response(self) - def validate_seg_dict(seg_dict, head_seg_resp): + def validate_seg_dict(seg_dict, head_seg_resp, allow_empty_segment): if not head_seg_resp.is_success: problem_segments.append([quote(obj_name), head_seg_resp.status]) @@ -976,7 +976,7 @@ class StaticLargeObject(object): seg_dict['range'] = '%d-%d' % (rng[0], rng[1] - 1) segment_length = rng[1] - rng[0] - if segment_length < 1: + if segment_length < 1 and not allow_empty_segment: problem_segments.append( [quote(obj_name), 'Too small; each segment must be at least 1 byte.']) @@ -1012,7 +1012,8 @@ class StaticLargeObject(object): (path, ) for path in path2indices)): for i in path2indices[obj_name]: segment_length, seg_data = validate_seg_dict( - parsed_data[i], resp) + parsed_data[i], resp, + allow_empty_segment=(i == len(parsed_data) - 1)) data_for_storage[i] = seg_data total_size += segment_length diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index a74fe6a08d..d0585276ee 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -487,17 +487,17 @@ class TestSloPutManifest(SloTestCase): status, headers, body = self.call_slo(req) self.assertEqual(status, '400 Bad Request') - def test_handle_multipart_put_disallow_empty_last_segment(self): + def test_handle_multipart_put_allow_empty_last_segment(self): test_json_data = json.dumps([{'path': '/cont/object', 'etag': 'etagoftheobjectsegment', 'size_bytes': 100}, - {'path': '/cont/small_object', + {'path': '/cont/empty_object', 'etag': 'etagoftheobjectsegment', 'size_bytes': 0}]) - req = Request.blank('/v1/a/c/o?multipart-manifest=put', + req = Request.blank('/v1/AUTH_test/c/man?multipart-manifest=put', method='PUT', body=test_json_data) status, headers, body = self.call_slo(req) - self.assertEqual(status, '400 Bad Request') + self.assertEqual(status, '201 Created') def test_handle_multipart_put_success_unicode(self): test_json_data = json.dumps([{'path': u'/cont/object\u2661',