This server could not ' 'verif...) while retrieving /v1/AUTH_test/gettest/c_15' ]) self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), # This one has the error, and so is the last one we fetch. ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')]) def test_error_fetching_submanifest(self): self.app.register('GET', '/v1/AUTH_test/gettest/manifest-bc', swob.HTTPUnauthorized, {}, None) req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual("200 OK", status) self.assertEqual(b"aaaaa", body) self.assertEqual(self.slo.logger.get_lines_for_level('error'), [ 'while fetching /v1/AUTH_test/gettest/manifest-abcd, GET of ' 'submanifest /v1/AUTH_test/gettest/manifest-bc failed with ' 'status 401 (
This server could ' 'not verif...)' ]) self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), # TODO: skip coalecse until validate to re-order sub-manifest req # This one has the error, and so is the last one we fetch. ('GET', '/v1/AUTH_test/gettest/manifest-bc'), # But we were looking ahead to see if we could combine ranges, # so we still get the first segment out ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get')]) def test_error_fetching_first_segment_submanifest(self): # This differs from the normal submanifest error because this one # happens before we've actually sent any response body. self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-a', swob.HTTPForbidden, {}, None) self._setup_manifest('manifest-a', [ {'name': '/gettest/manifest-a', 'sub_slo': True, 'content_type': 'application/json', 'hash': 'manifest-a', 'bytes': '12345'}, ], container='gettest') req = Request.blank( '/v1/AUTH_test/gettest/manifest-manifest-a', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual('409 Conflict', status) self.assertEqual(self.slo.logger.get_lines_for_level('error'), [ 'while fetching /v1/AUTH_test/gettest/manifest-manifest-a, GET ' 'of submanifest /v1/AUTH_test/gettest/manifest-a failed with ' 'status 403 (
Access was denied to ' 'this reso...)' ]) def test_invalid_json_submanifest(self): self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-bc', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true', 'X-Object-Meta-Plant': 'Ficus'}, "[this {isn't (JSON") req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual('200 OK', status) self.assertEqual(body, b'aaaaa') lines = self.slo.logger.get_lines_for_level('error') self.assertEqual(lines, [ mock.ANY, 'while fetching /v1/AUTH_test/gettest/manifest-abcd, ' 'JSON-decoding of submanifest /v1/AUTH_test/gettest/manifest-bc ' 'failed with 500 Internal Error' ]) self.assertIn(lines[0], [ # py2 'Unable to load SLO manifest: ' 'No JSON object could be decoded', # py3 'Unable to load SLO manifest: ' 'Expecting value: line 1 column 2 (char 1)', ]) def test_mismatched_etag(self): self._setup_manifest('a-b-badetag-c', [ {'name': '/gettest/a_5', 'hash': md5hex('a' * 5), 'content_type': 'text/plain', 'bytes': '5'}, {'name': '/gettest/b_10', 'hash': 'wrong!', 'content_type': 'text/plain', 'bytes': '10'}, {'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'content_type': 'text/plain', 'bytes': '15'}, ], container='gettest') req = Request.blank( '/v1/AUTH_test/gettest/manifest-a-b-badetag-c', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual('200 OK', status) self.assertEqual(headers['Etag'], '"%s"' % self.manifest_a_b_badetag_c_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_a_b_badetag_c_json_md5) self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(body, b'aaaaa') self.assertEqual(self.slo.logger.get_lines_for_level('error'), [ 'Object segment no longer valid: /v1/AUTH_test/gettest/b_10 ' 'etag: 82136b4240d6ce4ea7d03e51469a393b != wrong! or 10 != 10.' ]) self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/gettest/manifest-a-b-badetag-c'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ]) # we don't drain the segment's resp_iter if validation fails self.expected_unread_requests[ ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get')] = 1 def test_mismatched_size(self): self._setup_manifest('a-b-badsize-c', [ {'name': '/gettest/a_5', 'hash': md5hex('a' * 5), 'content_type': 'text/plain', 'bytes': '5'}, {'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'content_type': 'text/plain', 'bytes': '999999'}, {'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'content_type': 'text/plain', 'bytes': '15'}, ], container='gettest') req = Request.blank( '/v1/AUTH_test/gettest/manifest-a-b-badsize-c', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual('200 OK', status) self.assertEqual(body, b'aaaaa') self.assertEqual(self.slo.logger.get_lines_for_level('error'), [ 'Object segment no longer valid: /v1/AUTH_test/gettest/b_10 ' 'etag: 82136b4240d6ce4ea7d03e51469a393b != ' '82136b4240d6ce4ea7d03e51469a393b or 10 != 999999.' ]) # we don't drain the segment's resp_iter if validation fails self.expected_unread_requests[ ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get')] = 1 def test_mismatched_checksum(self): self.app.register( 'GET', '/v1/AUTH_test/gettest/a_5', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex('a' * 5)}, # this segment has invalid content 'x' * 5) self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'content_type': 'text/plain', 'bytes': '10'}, {'name': '/gettest/a_5', 'hash': md5hex('a' * 5), 'content_type': 'text/plain', 'bytes': '5'}, {'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'content_type': 'text/plain', 'bytes': '15'}])) req = Request.blank('/v1/AUTH_test/gettest/manifest') status, headers, body = self.call_slo(req) self.assertEqual('200 OK', status) self.assertEqual(body, (b'b' * 10 + b'x' * 5)) self.assertEqual(self.slo.logger.get_lines_for_level('error'), [ 'Bad MD5 checksum for /v1/AUTH_test/gettest/a_5 as part of ' '/v1/AUTH_test/gettest/manifest: headers had ' '594f803b380a41396ed63dca39503542, but object MD5 was ' 'actually fb0e22c79ac75679e9881e6ba183b354', ]) def test_mismatched_length(self): self.app.register( 'GET', '/v1/AUTH_test/gettest/a_5', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex('a' * 5)}, # this segment comes up short [b'a' * 4]) self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'content_type': 'text/plain', 'bytes': '10'}, {'name': '/gettest/a_5', 'hash': md5hex('a' * 5), 'content_type': 'text/plain', 'bytes': '5'}, {'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'content_type': 'text/plain', 'bytes': '15'}])) req = Request.blank('/v1/AUTH_test/gettest/manifest') status, headers, body = self.call_slo(req) self.assertEqual('200 OK', status) self.assertEqual(body, (b'b' * 10 + b'a' * 4)) self.assertEqual(self.slo.logger.get_lines_for_level('error'), [ 'Bad response length for /v1/AUTH_test/gettest/a_5 as part of ' '/v1/AUTH_test/gettest/manifest: headers had 5, but ' 'response length was actually 4', ]) def test_first_segment_mismatched_etag(self): req = Request.blank('/v1/AUTH_test/gettest/manifest-badetag') status, headers, body = self.call_slo(req) self.assertEqual('409 Conflict', status) self.assertNotIn('Etag', headers) self.assertNotIn('X-Manifest-Etag', headers) self.assertNotIn('X-Static-Large-Object', headers) self.assertEqual(int(headers['Content-Length']), len(body)) self.assertIn(b'There was a conflict', body) self.assertEqual(self.slo.logger.get_lines_for_level('error'), [ 'Object segment no longer valid: /v1/AUTH_test/gettest/a_5 ' 'etag: 594f803b380a41396ed63dca39503542 != wrong! or 5 != 5.' ]) self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/gettest/manifest-badetag'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ]) # we don't drain the segment's resp_iter if validation fails self.expected_unread_requests[ ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get')] = 1 def test_head_does_not_validate_first_segment_mismatched_etag(self): req = Request.blank('/v1/AUTH_test/gettest/manifest-badetag', method='HEAD') status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_badetag_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_badetag_json_md5) self.assertEqual(int(headers['Content-Length']), self.manifest_badetag_slo_size) self.assertEqual(headers['X-Static-Large-Object'], 'true') expected_calls = [ ('HEAD', '/v1/AUTH_test/gettest/manifest-badetag'), ] if not self.modern_manifest_headers: expected_calls.append( ('GET', '/v1/AUTH_test/gettest/manifest-badetag')) self.assertEqual(self.app.calls, expected_calls) def test_first_segment_mismatched_size(self): req = Request.blank('/v1/AUTH_test/gettest/manifest-badsize', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual('409 Conflict', status) self.assertNotIn('Etag', headers) self.assertNotIn('X-Manifest-Etag', headers) self.assertNotIn('X-Static-Large-Object', headers) self.assertEqual(int(headers['Content-Length']), len(body)) self.assertIn(b'There was a conflict', body) self.assertEqual(self.slo.logger.get_lines_for_level('error'), [ 'Object segment no longer valid: /v1/AUTH_test/gettest/a_5 ' 'etag: 594f803b380a41396ed63dca39503542 != ' '594f803b380a41396ed63dca39503542 or 5 != 999999.' ]) self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/gettest/manifest-badsize'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ]) # we don't drain the segment's resp_iter if validation fails self.expected_unread_requests[ ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get')] = 1 def test_head_does_not_validate_first_segment_mismatched_size(self): req = Request.blank('/v1/AUTH_test/gettest/manifest-badsize', method='HEAD') status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_badsize_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_badsize_json_md5) self.assertEqual(int(headers['Content-Length']), self.manifest_badsize_slo_size) self.assertEqual(headers['X-Static-Large-Object'], 'true') expected_calls = [ ('HEAD', '/v1/AUTH_test/gettest/manifest-badsize'), ] if not self.modern_manifest_headers: expected_calls.append( ('GET', '/v1/AUTH_test/gettest/manifest-badsize')) self.assertEqual(self.app.calls, expected_calls) @patch('swift.common.request_helpers.time') def test_download_takes_too_long(self, mock_time): mock_time.time.side_effect = [ 0, # start time 10 * 3600, # a_5 20 * 3600, # b_10 30 * 3600, # c_15, but then we time out ] req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(self.slo.logger.get_lines_for_level('error'), [ 'While processing manifest /v1/AUTH_test/gettest/manifest-abcd, ' 'max LO GET time of 86400s exceeded' ]) self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')]) # we timeout without reading the whole of the last segment self.expected_unread_requests[ ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')] = 1 def test_first_segment_not_exists(self): self.app.register('GET', '/v1/AUTH_test/gettest/not_exists_obj', swob.HTTPNotFound, {}, None) self.app.register('GET', '/v1/AUTH_test/gettest/manifest-not-exists', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/gettest/not_exists_obj', 'hash': md5hex('not_exists_obj'), 'content_type': 'text/plain', 'bytes': '%d' % len('not_exists_obj') }])) req = Request.blank('/v1/AUTH_test/gettest/manifest-not-exists', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual('409 Conflict', status) self.assertEqual(self.slo.logger.get_lines_for_level('error'), [ 'While processing manifest /v1/AUTH_test/gettest/' 'manifest-not-exists, got 404 (
The ' 'resource could not be foun...) while retrieving /v1/AUTH_test/' 'gettest/not_exists_obj' ]) def test_first_segment_not_available(self): self.app.register('GET', '/v1/AUTH_test/gettest/not_avail_obj', swob.HTTPServiceUnavailable, {}, None) self.app.register('GET', '/v1/AUTH_test/gettest/manifest-not-avail', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/gettest/not_avail_obj', 'hash': md5hex('not_avail_obj'), 'content_type': 'text/plain', 'bytes': '%d' % len('not_avail_obj') }])) req = Request.blank('/v1/AUTH_test/gettest/manifest-not-avail', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual('503 Service Unavailable', status) self.assertEqual(self.slo.logger.get_lines_for_level('error'), [ 'While processing manifest /v1/AUTH_test/gettest/' 'manifest-not-avail, got 503 (
The server is curren...) while retrieving /v1/AUTH_test/' 'gettest/not_avail_obj' ]) self.assertIn(b'Service Unavailable', body) class TestSloErrorsOldManifests(TestSloErrors): modern_manifest_headers = False class TestSloDataSegments(SloGETorHEADTestCase): # data segments were added some months after modern slo-sysmeta def setUp(self): super(TestSloDataSegments, self).setUp() self._setup_alphabet_objects('ab') def test_leading_data_segment(self): slo_etag = md5hex( md5hex('preamble') + md5hex('a' * 5) ) preamble = base64.b64encode(b'preamble') self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-single-preamble', swob.HTTPOk, { 'Content-Type': 'application/json', 'X-Static-Large-Object': 'true' }, json.dumps([{ 'data': preamble.decode('ascii') }, { 'name': '/gettest/a_5', 'hash': md5hex('a' * 5), 'content_type': 'text/plain', 'bytes': '5', }]) ) req = Request.blank( '/v1/AUTH_test/gettest/manifest-single-preamble', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual('200 OK', status) self.assertEqual(body, b'preambleaaaaa') self.assertEqual(headers['Etag'], '"%s"' % slo_etag) self.assertEqual(headers['Content-Length'], '13') def test_trailing_data_segment(self): slo_etag = md5hex( md5hex('a' * 5) + md5hex('postamble') ) postamble = base64.b64encode(b'postamble') self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-single-postamble', swob.HTTPOk, { 'Content-Type': 'application/json', 'X-Static-Large-Object': 'true' }, json.dumps([{ 'name': '/gettest/a_5', 'hash': md5hex('a' * 5), 'content_type': 'text/plain', 'bytes': '5', }, { 'data': postamble.decode('ascii') }]).encode('ascii') ) req = Request.blank( '/v1/AUTH_test/gettest/manifest-single-postamble', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual('200 OK', status) self.assertEqual(body, b'aaaaapostamble') self.assertEqual(headers['Etag'], '"%s"' % slo_etag) self.assertEqual(headers['Content-Length'], '14') def test_data_segment_sandwich(self): slo_etag = md5hex( md5hex('preamble') + md5hex('a' * 5) + md5hex('postamble') ) preamble = base64.b64encode(b'preamble') postamble = base64.b64encode(b'postamble') self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-single-prepostamble', swob.HTTPOk, { 'Content-Type': 'application/json', 'X-Static-Large-Object': 'true' }, json.dumps([{ 'data': preamble.decode('ascii'), }, { 'name': '/gettest/a_5', 'hash': md5hex('a' * 5), 'content_type': 'text/plain', 'bytes': '5', }, { 'data': postamble.decode('ascii') }]) ) # Test the whole SLO req = Request.blank( '/v1/AUTH_test/gettest/manifest-single-prepostamble', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual('200 OK', status) self.assertEqual(body, b'preambleaaaaapostamble') self.assertEqual(headers['Etag'], '"%s"' % slo_etag) self.assertEqual(headers['Content-Length'], '22') # Test complete preamble only req = Request.blank( '/v1/AUTH_test/gettest/manifest-single-prepostamble', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=0-7'}) status, headers, body = self.call_slo(req) self.assertEqual('206 Partial Content', status) self.assertEqual(body, b'preamble') # Test range within preamble only req = Request.blank( '/v1/AUTH_test/gettest/manifest-single-prepostamble', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=1-5'}) status, headers, body = self.call_slo(req) self.assertEqual('206 Partial Content', status) self.assertEqual(body, b'reamb') # Test complete postamble only req = Request.blank( '/v1/AUTH_test/gettest/manifest-single-prepostamble', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=13-21'}) status, headers, body = self.call_slo(req) self.assertEqual('206 Partial Content', status) self.assertEqual(body, b'postamble') # Test partial pre and postamble req = Request.blank( '/v1/AUTH_test/gettest/manifest-single-prepostamble', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=4-16'}) status, headers, body = self.call_slo(req) self.assertEqual('206 Partial Content', status) self.assertEqual(body, b'mbleaaaaapost') # Test partial preamble and first byte of data req = Request.blank( '/v1/AUTH_test/gettest/manifest-single-prepostamble', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=1-8'}) status, headers, body = self.call_slo(req) self.assertEqual('206 Partial Content', status) self.assertEqual(body, b'reamblea') # Test last byte of segment data and partial postamble req = Request.blank( '/v1/AUTH_test/gettest/manifest-single-prepostamble', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=12-16'}) status, headers, body = self.call_slo(req) self.assertEqual('206 Partial Content', status) self.assertEqual(body, b'apost') def test_bunches_of_data_segments(self): slo_etag = md5hex( md5hex('ABCDEF') + md5hex('a' * 5) + md5hex('123456') + md5hex('GHIJKL') + md5hex('b' * 10) + md5hex('7890@#') ) self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-multi-prepostamble', swob.HTTPOk, { 'Content-Type': 'application/json', 'X-Static-Large-Object': 'true' }, json.dumps([ { 'data': base64.b64encode(b'ABCDEF').decode('ascii') }, { 'name': '/gettest/a_5', 'hash': md5hex('a' * 5), 'content_type': 'text/plain', 'bytes': '5', }, { 'data': base64.b64encode(b'123456').decode('ascii') }, { 'data': base64.b64encode(b'GHIJKL').decode('ascii') }, { 'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'content_type': 'text/plain', 'bytes': '10', }, { 'data': base64.b64encode(b'7890@#').decode('ascii') } ]) ) # Test the whole SLO req = Request.blank( '/v1/AUTH_test/gettest/manifest-multi-prepostamble', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual('200 OK', status) self.assertEqual(body, b'ABCDEFaaaaa123456GHIJKLbbbbbbbbbb7890@#') self.assertEqual(headers['Etag'], '"%s"' % slo_etag) self.assertEqual(headers['Content-Length'], '39') # Test last byte first pre-amble to first byte of second postamble req = Request.blank( '/v1/AUTH_test/gettest/manifest-multi-prepostamble', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=5-33'}) status, headers, body = self.call_slo(req) self.assertEqual('206 Partial Content', status) self.assertEqual(body, b'Faaaaa123456GHIJKLbbbbbbbbbb7') # Test only second complete preamble req = Request.blank( '/v1/AUTH_test/gettest/manifest-multi-prepostamble', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=17-22'}) status, headers, body = self.call_slo(req) self.assertEqual('206 Partial Content', status) self.assertEqual(body, b'GHIJKL') # Test only first complete postamble req = Request.blank( '/v1/AUTH_test/gettest/manifest-multi-prepostamble', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=11-16'}) status, headers, body = self.call_slo(req) self.assertEqual('206 Partial Content', status) self.assertEqual(body, b'123456') # Test only range within first postamble req = Request.blank( '/v1/AUTH_test/gettest/manifest-multi-prepostamble', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=12-15'}) status, headers, body = self.call_slo(req) self.assertEqual('206 Partial Content', status) self.assertEqual(body, b'2345') # Test only range within first postamble and second preamble req = Request.blank( '/v1/AUTH_test/gettest/manifest-multi-prepostamble', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=12-18'}) status, headers, body = self.call_slo(req) self.assertEqual('206 Partial Content', status) self.assertEqual(body, b'23456GH') class TestSloConditionalGetOldManifest(SloGETorHEADTestCase): modern_manifest_headers = False def setUp(self): super(TestSloConditionalGetOldManifest, self).setUp() self._setup_alphabet_objects('abcd') self._setup_manifest_bc() self._setup_manifest_abcd() # plain object with alt-etag num_segments = 2 alt_seg_info = [] for i in range(num_segments): body = (b'alt_%02d' % i) * 5 etag = md5hex(body) self.app.register( 'GET', '/v1/AUTH_test/c/alt_%02d' % i, swob.HTTPOk, { 'Content-Length': len(body), 'Etag': etag, 'X-Object-Sysmeta-Alt-Etag': 'seg-etag-%02d' % i }, body=body) alt_seg_info.append((body, etag)) # s3api is to the left of SLO and writes an alternate etag for # conditional requests to match self._setup_manifest('alt', [{ 'name': '/c/alt_%02d' % i, 'bytes': len(body), 'hash': etag, 'content_type': 'text/plain', } for i, (body, etag) in enumerate(alt_seg_info)], extra_headers={ 'X-Object-Sysmeta-Alt-Etag': '"alt-etag-1"', }) self._setup_manifest('last-modified', [ {'name': '/gettest/a_5', 'hash': md5hex('a' * 5), 'bytes': '5', 'content_type': 'text/plain'}, {'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'bytes': '15', 'content_type': 'text/plain'}, ], extra_headers={ 'Last-Modified': 'Mon, 23 Oct 2023 10:05:32 GMT', }) def test_if_none_match_matches(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', headers={'If-None-Match': self.manifest_abcd_slo_etag}) status, headers, body = self.call_slo(req) self.assertEqual(status, '304 Not Modified') self.assertEqual('"%s"' % self.manifest_abcd_slo_etag, headers['Etag']) # conditional errors are always zero-bytes to save client egress self.assertEqual('0', headers['Content-Length']) self.assertEqual(b'', body) expected_app_calls = [('GET', '/v1/AUTH_test/gettest/manifest-abcd')] if not self.modern_manifest_headers: expected_app_calls.extend([ # N.B. since manifest didn't match slo_etag => no refetch # TODO: skip coalecse until validate to avoid sub-manifest req ('GET', '/v1/AUTH_test/gettest/manifest-bc'), # for legacy manifests we don't know if swob will return a # successful response or conditional error so we validate the # first segment to avoid a 2XX when we should 5XX ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ]) # when swob decides to error it closes our SegmentedIterable # and we don't drain the (possibly large) segment. self.expected_unread_requests[('GET', '/v1/AUTH_test/gettest/a_5' '?multipart-manifest=get')] = 1 self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual(self.app.headers[0].get('X-Backend-Etag-Is-At'), 'x-object-sysmeta-slo-etag') if not self.modern_manifest_headers: for headers in self.app.headers[1:]: self.assertNotIn('If-Match', headers) self.assertNotIn('X-Backend-Etag-Is-At', headers) def test_if_none_match_mismatches(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', headers={'If-None-Match': "not-%s" % self.manifest_abcd_slo_etag}) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual('"%s"' % self.manifest_abcd_slo_etag, headers['Etag']) self.assertEqual('50', headers['Content-Length']) self.assertEqual( body, b'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd') expected_app_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get'), ] self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual(self.app.headers[0].get('X-Backend-Etag-Is-At'), 'x-object-sysmeta-slo-etag') def test_if_none_match_mismatches_json_md5(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', headers={'If-None-Match': self.manifest_abcd_json_md5}) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual('"%s"' % self.manifest_abcd_slo_etag, headers['Etag']) self.assertEqual('50', headers['Content-Length']) expected_app_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ] if not self.modern_manifest_headers: # w/o modern manifest headers, the json manifest etag responds 304 # and triggers a refetch! expected_app_calls.append( ('GET', '/v1/AUTH_test/gettest/manifest-abcd') ) expected_app_calls.extend([ ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get'), ]) self.assertEqual(self.app.calls, expected_app_calls) def test_if_none_match_matches_alternate_etag(self): req = Request.blank( '/v1/AUTH_test/c/manifest-alt', headers={'If-None-Match': '"alt-etag-1"'}) update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '304 Not Modified') # N.B. Etag-Is-At only effects conditional matching, not response Etag self.assertEqual('"%s"' % self.manifest_alt_slo_etag, headers['Etag']) # ... but the response Sysmeta will be available to wrapping middleware self.assertEqual('"alt-etag-1"', headers['X-Object-Sysmeta-Alt-Etag']) # conditional errors are always zero-bytes to save client egress self.assertEqual('0', headers['Content-Length']) self.assertEqual(b'', body) expected_app_calls = [('GET', '/v1/AUTH_test/c/manifest-alt')] self.assertEqual( self.app.headers[0].get('X-Backend-Etag-Is-At'), 'X-Object-Sysmeta-Alt-Etag,x-object-sysmeta-slo-etag') if not self.modern_manifest_headers: expected_app_calls.extend([ # Needed to re-fetch because if-match can't find slo-etag, and # has to 304 ('GET', '/v1/AUTH_test/c/manifest-alt'), # for legacy manifests we don't know if swob will return a # successful response or conditional error so we validate the # first segment to avoid a 2XX when we should 5XX ('GET', '/v1/AUTH_test/c/alt_00?multipart-manifest=get'), ]) # when swob decides to error it closes our SegmentedIterable # and we don't drain the (possibly large) segment. self.expected_unread_requests[('GET', '/v1/AUTH_test/c/alt_00' '?multipart-manifest=get')] = 1 for headers in self.app.headers[1:]: self.assertNotIn('If-None-Match', headers) self.assertNotIn('X-Backend-Etag-Is-At', headers) self.assertEqual(self.app.calls, expected_app_calls) def test_if_none_match_matches_no_alternate_etag(self): # this is similar to test_if_none_match_matches, but serves as a sanity # check to test_if_none_match_mismatches_alternate_etag, which appends # to etag-is-at req = Request.blank( '/v1/AUTH_test/c/manifest-alt', headers={'If-None-Match': self.manifest_alt_slo_etag}) status, headers, body = self.call_slo(req) self.assertEqual(status, '304 Not Modified') self.assertEqual('"%s"' % self.manifest_alt_slo_etag, headers['Etag']) # conditional errors are always zero-bytes to save client egress self.assertEqual('0', headers['Content-Length']) self.assertEqual(b'', body) expected_app_calls = [('GET', '/v1/AUTH_test/c/manifest-alt')] self.assertEqual(self.app.headers[0].get('X-Backend-Etag-Is-At'), 'x-object-sysmeta-slo-etag') if not self.modern_manifest_headers: expected_app_calls.extend([ # N.B. since manifest didn't match slo_etag => no refetch # for legacy manifests we don't know if swob will return a # successful response or conditional error so we validate the # first segment to avoid a 2XX when we should 5XX ('GET', '/v1/AUTH_test/c/alt_00?multipart-manifest=get'), ]) for headers in self.app.headers[1:]: self.assertNotIn('If-Match', headers) self.assertNotIn('X-Backend-Etag-Is-At', headers) # for legacy manifests we don't know if swob will return a # successful response or conditional error so we validate the first # segment to avoid a 2XX when we should 5XX, when swob decides to # error it closes our SegmentedIterable and we don't drain the # (possibly large) segment. self.expected_unread_requests[ ('GET', '/v1/AUTH_test/c/alt_00' '?multipart-manifest=get')] = 1 self.assertEqual(self.app.calls, expected_app_calls) def test_if_none_match_mismatches_alternate_etag(self): req = Request.blank( '/v1/AUTH_test/c/manifest-alt', headers={'If-None-Match': self.manifest_alt_slo_etag}) # N.B. SLO request with if-none-match slo_etag would normally respond # not modified (see test_if_none_match_matches_no_alternate_etag), but # here we provide alt-tag so it doesn't match so the request is success update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') # N.B. Etag-Is-At only effects conditional matching, not response Etag self.assertEqual('"%s"' % self.manifest_alt_slo_etag, headers['Etag']) # ... but the response Sysmeta will be available to wrapping middleware self.assertEqual('"alt-etag-1"', headers['X-Object-Sysmeta-Alt-Etag']) self.assertEqual(self.manifest_alt_slo_size, int(headers['Content-Length'])) expected_app_calls = [ ('GET', '/v1/AUTH_test/c/manifest-alt'), ('GET', '/v1/AUTH_test/c/alt_00?multipart-manifest=get'), ('GET', '/v1/AUTH_test/c/alt_01?multipart-manifest=get'), ] self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual( self.app.headers[0].get('X-Backend-Etag-Is-At'), 'X-Object-Sysmeta-Alt-Etag,x-object-sysmeta-slo-etag') for headers in self.app.headers[1:]: self.assertNotIn('If-None-Match', headers) self.assertNotIn('X-Backend-Etag-Is-At', headers) def test_if_match_matches(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': self.manifest_abcd_slo_etag}) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual('"%s"' % self.manifest_abcd_slo_etag, headers['Etag']) self.assertEqual('50', headers['Content-Length']) self.assertEqual( body, b'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd') expected_app_calls = [('GET', '/v1/AUTH_test/gettest/manifest-abcd')] if not self.modern_manifest_headers: # Manifest never matches -> got back a 412; need to re-fetch expected_app_calls.append( ('GET', '/v1/AUTH_test/gettest/manifest-abcd')) expected_app_calls.extend([ ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get'), ]) self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual(self.app.headers[0].get('X-Backend-Etag-Is-At'), 'x-object-sysmeta-slo-etag') def test_if_match_mismatches(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', headers={'If-Match': 'not-%s' % self.manifest_abcd_json_md5}) status, headers, body = self.call_slo(req) self.assertEqual(status, '412 Precondition Failed') self.assertEqual('"%s"' % self.manifest_abcd_slo_etag, headers['Etag']) # conditional errors are always zero-bytes to save client egress self.assertEqual('0', headers['Content-Length']) self.assertEqual(b'', body) expected_app_calls = [('GET', '/v1/AUTH_test/gettest/manifest-abcd')] if not self.modern_manifest_headers: expected_app_calls.extend([ # Manifest "never" matches -> got back a 412; need to re-fetch ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), # TODO: skip coalecse until validate to avoid sub-manifest req ('GET', '/v1/AUTH_test/gettest/manifest-bc'), # for legacy manifests we don't know if swob will return a # successful response or conditional error so we validate the # first segment to avoid a 2XX when we should 5XX ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ]) # when swob decides to error it closes our SegmentedIterable # and we don't drain the (possibly large) segment. self.expected_unread_requests[ ('GET', '/v1/AUTH_test/gettest/a_5' '?multipart-manifest=get')] = 1 self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual(self.app.headers[0].get('X-Backend-Etag-Is-At'), 'x-object-sysmeta-slo-etag') if not self.modern_manifest_headers: for headers in self.app.headers[1:]: self.assertNotIn('If-Match', headers) self.assertNotIn('X-Backend-Etag-Is-At', headers) def test_if_match_mismatches_manifest_json_md5(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', headers={'If-Match': self.manifest_abcd_json_md5}) status, headers, body = self.call_slo(req) self.assertEqual(status, '412 Precondition Failed') self.assertEqual('"%s"' % self.manifest_abcd_slo_etag, headers['Etag']) # 412 is always zero-bytes because client is trying to save egress self.assertEqual('0', headers['Content-Length']) self.assertEqual(body, b'') expected_app_calls = [('GET', '/v1/AUTH_test/gettest/manifest-abcd')] self.assertEqual(self.app.headers[0].get('X-Backend-Etag-Is-At'), 'x-object-sysmeta-slo-etag') if not self.modern_manifest_headers: # We *still* verify the first segment expected_app_calls.extend([ # N.B. since manifest matched => no refetch # TODO: skip coalecse until validate to avoid sub-manifest req ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ]) # swob conditional errors throw away the SegmentedIterable w/o # reading the remaining segments self.expected_unread_requests[('GET', '/v1/AUTH_test/gettest/a_5' '?multipart-manifest=get')] = 1 self.assertEqual(self.app.calls, expected_app_calls) def test_if_match_matches_alternate_etag(self): req = Request.blank( '/v1/AUTH_test/c/manifest-alt', headers={'If-Match': '"alt-etag-1"'}) update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(self.manifest_alt_slo_size, int(headers['Content-Length'])) # N.B. Etag-Is-At only effects conditional matching, not response Etag self.assertEqual('"%s"' % self.manifest_alt_slo_etag, headers['Etag']) # ... but the response Sysmeta will be available to wrapping middleware self.assertEqual('"alt-etag-1"', headers['X-Object-Sysmeta-Alt-Etag']) expected_app_calls = [ ('GET', '/v1/AUTH_test/c/manifest-alt'), ('GET', '/v1/AUTH_test/c/alt_00?multipart-manifest=get'), ('GET', '/v1/AUTH_test/c/alt_01?multipart-manifest=get'), ] self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual( self.app.headers[0].get('X-Backend-Etag-Is-At'), 'X-Object-Sysmeta-Alt-Etag,x-object-sysmeta-slo-etag') self.assertNotIn('X-Backend-Etag-Is-At', self.app.headers[1]) def test_if_match_mismatches_alternate_etag(self): req = Request.blank( '/v1/AUTH_test/c/manifest-alt', headers={'If-Match': self.manifest_alt_slo_etag}) update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '412 Precondition Failed') # N.B. Etag-Is-At only effects conditional matching, not response Etag self.assertEqual('"%s"' % self.manifest_alt_slo_etag, headers['Etag']) # ... but the response Sysmeta will be available to wrapping middleware self.assertEqual('"alt-etag-1"', headers['X-Object-Sysmeta-Alt-Etag']) # conditional errors are always zero-bytes to save client egress self.assertEqual('0', headers['Content-Length']) self.assertEqual(b'', body) expected_app_calls = [('GET', '/v1/AUTH_test/c/manifest-alt')] self.assertEqual(self.app.headers[0].get('X-Backend-Etag-Is-At'), 'X-Object-Sysmeta-Alt-Etag,x-object-sysmeta-slo-etag') if not self.modern_manifest_headers: expected_app_calls.extend([ # Needed to re-fetch because if-match can't find slo-etag ('GET', '/v1/AUTH_test/c/manifest-alt'), # We end up validating the first segment ('GET', '/v1/AUTH_test/c/alt_00?multipart-manifest=get'), ]) # for legacy manifests we don't know if swob will return a # successful response or conditional error so we validate the first # segment to avoid a 2XX when we should 5XX, when swob decides to # error it closes our SegmentedIterable and we don't drain the # (possibly large) segment. self.expected_unread_requests[ ('GET', '/v1/AUTH_test/c/alt_00' '?multipart-manifest=get')] = 1 for headers in self.app.headers[1:]: self.assertNotIn('If-Match', headers) self.assertNotIn('X-Backend-Etag-Is-At', headers) self.assertEqual(self.app.calls, expected_app_calls) def test_manifest_get_if_none_match_matches(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd?multipart-manifest=get', headers={'If-None-Match': self.manifest_abcd_json_md5}) status, headers, body = self.call_slo(req) self.assertEqual(status, '304 Not Modified') self.assertEqual(self.manifest_abcd_json_md5, headers['Etag']) # conditional errors are always zero-bytes to save client egress self.assertEqual('0', headers['Content-Length']) self.assertEqual(b'', body) expected_app_calls = [('GET', '/v1/AUTH_test/gettest/manifest-abcd' '?multipart-manifest=get')] self.assertEqual(self.app.calls, expected_app_calls) self.assertNotIn('X-Backend-Etag-Is-At', self.app.headers[0]) def test_manifest_get_if_none_match_mismatches(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd?multipart-manifest=get', headers={'If-None-Match': "not-%s" % self.manifest_abcd_json_md5}) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(self.manifest_abcd_json_md5, headers['Etag']) self.assertEqual(self.manifest_abcd_json_size, int(headers['Content-Length'])) data = json.loads(body) self.assertEqual( ['/gettest/a_5', '/gettest/manifest-bc', '/gettest/d_20'], [s['name'] for s in data]) self.assertEqual(md5hex(body), self.manifest_abcd_json_md5) expected_app_calls = [('GET', '/v1/AUTH_test/gettest/manifest-abcd' '?multipart-manifest=get')] self.assertEqual(self.app.calls, expected_app_calls) self.assertNotIn('X-Backend-Etag-Is-At', self.app.headers[0]) def test_manifest_get_if_none_match_matches_alternate_etag(self): req = Request.blank( '/v1/AUTH_test/c/manifest-alt?multipart-manifest=get', headers={'If-None-Match': '"alt-etag-1"'}) # who would do this for a multipart-manifest=get requests? update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '304 Not Modified') # N.B. Etag-Is-At only effects conditional matching, not response Etag self.assertEqual(self.manifest_alt_json_md5, headers['Etag']) # ... but the response Sysmeta will be available to wrapping middleware self.assertEqual('"alt-etag-1"', headers['X-Object-Sysmeta-Alt-Etag']) # conditional errors are always zero-bytes to save client egress self.assertEqual('0', headers['Content-Length']) self.assertEqual(b'', body) expected_app_calls = [('GET', '/v1/AUTH_test/c/manifest-alt' '?multipart-manifest=get')] self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual( self.app.headers[0].get('X-Backend-Etag-Is-At'), 'X-Object-Sysmeta-Alt-Etag') def test_manifest_get_if_none_match_mismatches_alternate_etag(self): req = Request.blank( '/v1/AUTH_test/c/manifest-alt?multipart-manifest=get', headers={'If-None-Match': '"not-alt-etag-1"'}) # who would do this for a multipart-manifest=get requests? update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') # N.B. Etag-Is-At only effects conditional matching, not response Etag self.assertEqual(self.manifest_alt_json_md5, headers['Etag']) # ... but the response Sysmeta will be available to wrapping middleware self.assertEqual('"alt-etag-1"', headers['X-Object-Sysmeta-Alt-Etag']) data = json.loads(body) self.assertEqual(['/c/alt_%02d' % i for i in range(len(data))], [s['name'] for s in data]) self.assertEqual(md5hex(body), self.manifest_alt_json_md5) expected_app_calls = [('GET', '/v1/AUTH_test/c/manifest-alt' '?multipart-manifest=get')] self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual( self.app.headers[0].get('X-Backend-Etag-Is-At'), 'X-Object-Sysmeta-Alt-Etag') def test_manifest_get_if_match_matches(self): # use if-match condition and expect to match req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd?multipart-manifest=get', headers={'If-Match': self.manifest_abcd_json_md5}) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(self.manifest_abcd_json_md5, headers['Etag']) expected_app_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd?multipart-manifest=get')] self.assertEqual(self.app.calls, expected_app_calls) self.assertNotIn('X-Backend-Etag-Is-At', self.app.headers[0]) def test_manifest_get_if_match_mismatches(self): # use if-match condition and expect to mismatch req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd?multipart-manifest=get', headers={'If-Match': self.manifest_abcd_slo_etag}) status, headers, body = self.call_slo(req) self.assertEqual(status, '412 Precondition Failed') self.assertEqual(self.manifest_abcd_json_md5, headers['Etag']) # conditional errors are always zero-bytes to save client egress self.assertEqual('0', headers['Content-Length']) self.assertEqual(b'', body) expected_app_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd?multipart-manifest=get')] self.assertEqual(self.app.calls, expected_app_calls) self.assertNotIn('X-Backend-Etag-Is-At', self.app.headers[0]) def test_manifest_get_if_match_matches_alternate_etag(self): # use if-match condition with alt-etag and expect to match req = Request.blank( '/v1/AUTH_test/c/manifest-alt?multipart-manifest=get', headers={'If-Match': '"alt-etag-1"'}) update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(self.manifest_alt_json_md5, headers['Etag']) expected_app_calls = [ ('GET', '/v1/AUTH_test/c/manifest-alt?multipart-manifest=get')] self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual('X-Object-Sysmeta-Alt-Etag', self.app.headers[0]['X-Backend-Etag-Is-At']) def test_manifest_get_if_match_mismatches_alternate_etag(self): # mis-match alternate etag req = Request.blank( '/v1/AUTH_test/c/manifest-alt?multipart-manifest=get', headers={'If-Match': self.manifest_alt_json_md5}) update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '412 Precondition Failed') self.assertEqual(self.manifest_alt_json_md5, headers['Etag']) # conditional errors are always zero-bytes to save client egress self.assertEqual('0', headers['Content-Length']) self.assertEqual(b'', body) expected_app_calls = [ ('GET', '/v1/AUTH_test/c/manifest-alt?multipart-manifest=get')] self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual('X-Object-Sysmeta-Alt-Etag', self.app.headers[0].get('X-Backend-Etag-Is-At')) def test_manifest_get_if_match_mismatches_without_alternate_etag(self): # sanity, this is similar to the test_manifest_get_if_match_mismatches # but in this case our manifest *has* an alt-etag, but no-one tells the # object server to look for it req = Request.blank( '/v1/AUTH_test/c/manifest-alt?multipart-manifest=get', headers={'If-Match': '"alt-etag-1"'}) # update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '412 Precondition Failed') self.assertEqual(self.manifest_alt_json_md5, headers['Etag']) # conditional errors are always zero-bytes to save client egress self.assertEqual('0', headers['Content-Length']) self.assertEqual(b'', body) expected_app_calls = [ ('GET', '/v1/AUTH_test/c/manifest-alt?multipart-manifest=get')] self.assertEqual(self.app.calls, expected_app_calls) self.assertNotIn('X-Backend-Etag-Is-At', self.app.headers[0]) def test_manifest_get_if_match_mismatches_alternate_etag_miss(self): # sanity, this is similar to # test_manifest_get_if_match_mismatches_alternate_etag but in this case # our manifest doesn't HAVE an alt-etag, so the object server falls # back to match with manifest's json_md5 req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd?multipart-manifest=get', headers={'If-Match': self.manifest_abcd_json_md5}) update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') def test_if_match_matches_alternate_etag_non_slo(self): # match alternate etag req = Request.blank( '/v1/AUTH_test/c/alt_00', headers={'If-Match': 'seg-etag-00'}) update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') # N.B. Etag-Is-At only effects conditional matching, not response Etag self.assertEqual(md5hex('alt_00' * 5), headers['Etag']) # ... but the response Sysmeta will be available to wrapping middleware self.assertEqual('seg-etag-00', headers['X-Object-Sysmeta-Alt-Etag']) expected_calls = [ ('GET', '/v1/AUTH_test/c/alt_00'), ] self.assertEqual(self.app.calls, expected_calls) def test_if_match_mismatches_alternate_etag_non_slo(self): # mis-match alternate etag req = Request.blank( '/v1/AUTH_test/c/alt_00', headers={'If-Match': md5hex(b'alt_00' * 5)}) update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) self.assertEqual(status, '412 Precondition Failed') # N.B. Etag-Is-At only effects conditional matching, not response Etag self.assertEqual(md5hex('alt_00' * 5), headers['Etag']) # ... but the response Sysmeta will be available to wrapping middleware self.assertEqual('seg-etag-00', headers['X-Object-Sysmeta-Alt-Etag']) expected_calls = [ ('GET', '/v1/AUTH_test/c/alt_00'), ] self.assertEqual(self.app.calls, expected_calls) def test_if_match_matches_alternate_etag_non_slo_after_refetch(self): self.app.register_next_response( 'GET', '/v1/AUTH_test/c/manifest-alt', swob.HTTPOk, {'Content-Length': '25', 'Etag': md5hex('alt_1' * 5), # N.B. manifest-alt gets overwritten mid-flight! 'X-Backend-Timestamp': '2345', 'X-Object-Sysmeta-Alt-Etag': 'alt-object-etag'}, body=b'alt_1' * 5) req = Request.blank( '/v1/AUTH_test/c/manifest-alt', headers={'If-Match': 'alt-object-etag'}) update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) expected_app_calls = [('GET', '/v1/AUTH_test/c/manifest-alt')] # first request asks for match on alt-etag self.assertEqual('alt-object-etag', self.app.headers[0]['If-Match']) self.assertIn('X-Object-Sysmeta-Alt-Etag', self.app.headers[0]['X-Backend-Etag-Is-At']) if self.modern_manifest_headers: # and since the response includes modern sysmeta, slo trusts the # 412 w/o refetch self.assertEqual(status, '412 Precondition Failed') # N.B. if the first repsonse had included a matching # alt-object-etag in sysmeta we would have returned 200, see # test_if_match_matches_alternate_etag with "alt-etag-1" self.assertEqual('"%s"' % self.manifest_alt_slo_etag, headers['Etag']) self.assertEqual('"alt-etag-1"', headers['X-Object-Sysmeta-Alt-Etag']) else: # ... but lacking modern sysmeta, slo will refetch a 412 expected_app_calls.append( ('GET', '/v1/AUTH_test/c/manifest-alt') ) # ... w/o conditionals self.assertNotIn('If-Match', self.app.headers[1]) self.assertNotIn('X-Backend-Etag-Is-At', self.app.headers[1]) # and the reconstructed swob response will *match* self.assertEqual(status, '200 OK') # N.B. Etag-Is-At only effects conditional matching, # not response Etag self.assertEqual(md5hex('alt_1' * 5), headers['Etag']) # ... but the response Sysmeta will be available to # wrapping middleware self.assertEqual('alt-object-etag', headers['X-Object-Sysmeta-Alt-Etag']) self.assertEqual(self.app.calls, expected_app_calls) def test_if_match_mismatches_alternate_etag_non_slo_after_refetch(self): self.app.register_next_response( 'GET', '/v1/AUTH_test/c/manifest-alt', swob.HTTPOk, {'Content-Length': '25', 'Etag': md5hex('alt_1' * 5), # N.B. manifest-alt gets overwritten mid-flight! 'X-Backend-Timestamp': '2345', 'X-Object-Sysmeta-Alt-Etag': 'alt-object-etag'}, body=b'alt_1' * 5) req = Request.blank( '/v1/AUTH_test/c/manifest-alt', headers={'If-Match': md5hex('alt_1' * 5)}) update_etag_is_at_header(req, 'X-Object-Sysmeta-Alt-Etag') status, headers, body = self.call_slo(req) expected_app_calls = [('GET', '/v1/AUTH_test/c/manifest-alt')] # first request asks for (mis)match on alt-etag self.assertEqual(md5hex('alt_1' * 5), self.app.headers[0]['If-Match']) self.assertIn('X-Object-Sysmeta-Alt-Etag', self.app.headers[0]['X-Backend-Etag-Is-At']) if self.modern_manifest_headers: # and since the response includes modern sysmeta, slo trusts the # 412 w/o refetch self.assertEqual(status, '412 Precondition Failed') # N.B. the first repsonse included an alt-etag in sysmeta (i.e. # "alt-etag-1"), it just doesn't match either - see # test_if_match_mismatches_alternate_etag self.assertEqual('"%s"' % self.manifest_alt_slo_etag, headers['Etag']) self.assertEqual('"alt-etag-1"', headers['X-Object-Sysmeta-Alt-Etag']) else: # ... but lacking modern sysmeta, slo will refetch a 412 expected_app_calls.append( ('GET', '/v1/AUTH_test/c/manifest-alt') ) # ... w/o conditionals self.assertNotIn('If-Match', self.app.headers[1]) self.assertNotIn('X-Backend-Etag-Is-At', self.app.headers[1]) # and the reconstructed swob response will *not* match self.assertEqual(status, '412 Precondition Failed') # N.B. Etag-Is-At only effects conditional matching, # not response Etag self.assertEqual(md5hex('alt_1' * 5), headers['Etag']) # ... but the response Sysmeta will be available to # wrapping middleware self.assertEqual('alt-object-etag', headers['X-Object-Sysmeta-Alt-Etag']) # swob is converting the successful non-slo response to conditional # error and closing our unconditionally refetched resp_iter self.expected_unread_requests[ ('GET', '/v1/AUTH_test/c/manifest-alt')] = 1 self.assertEqual(self.app.calls, expected_app_calls) def test_if_match_matches_and_range(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': self.manifest_abcd_slo_etag, 'Range': 'bytes=3-6'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertIn('bytes 3-6/50', headers['Content-Range']) self.assertEqual('"%s"' % self.manifest_abcd_slo_etag, headers['Etag']) self.assertEqual('4', headers['Content-Length']) self.assertEqual(body, b'aabb') expected_app_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ] if not self.modern_manifest_headers: # Needed to re-fetch because if-match can't find slo-etag expected_app_calls.append( ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ) # and then fetch the segments expected_app_calls.extend([ ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ]) self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual(self.app.headers[0].get('X-Backend-Etag-Is-At'), 'x-object-sysmeta-slo-etag') def test_old_swift_if_match_matches_and_range(self): self.app.can_ignore_range = False req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', headers={'If-Match': self.manifest_abcd_slo_etag, 'Range': 'bytes=3-6'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertEqual('"%s"' % self.manifest_abcd_slo_etag, headers['Etag']) self.assertEqual('4', headers['Content-Length']) self.assertEqual(body, b'aabb') expected_app_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), # new-sytle manifest sysmeta was added 2016, but ignore-range # didn't get added until 2020, so both new and old manifest # will still require refetch with old-swift ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ] self.assertEqual(self.app.calls, expected_app_calls) self.assertEqual(self.app.headers[0].get('X-Backend-Etag-Is-At'), 'x-object-sysmeta-slo-etag') def test_range_resume_download(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=20-'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertEqual(body, b'ccccccccccdddddddddddddddddddd') def test_get_with_if_modified_since(self): req = swob.Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Modified-Since': 'Wed, 12 Feb 2014 22:24:52 GMT', 'If-Unmodified-Since': 'Thu, 13 Feb 2014 23:25:53 GMT'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_slo_etag) self.assertEqual(int(headers['Content-Length']), self.manifest_abcd_slo_size) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_abcd_json_md5) self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(self.slo.logger.get_lines_for_level('error'), []) self.assertEqual( self.app.calls, [('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get')]) # It's important not to pass the If-[Un]Modified-Since header to the # proxy for segment or submanifest GET requests, as it may result in # 304 Not Modified responses, and those don't contain any useful data. for _, _, hdrs in self.app.calls_with_headers[1:]: self.assertNotIn('If-Modified-Since', hdrs) self.assertNotIn('If-Unmodified-Since', hdrs) def test_if_modified_since_ancient_date(self): req = swob.Request.blank( '/v1/AUTH_test/c/manifest-last-modified', headers={ 'If-Modified-Since': 'Fri, 01 Feb 2012 20:38:36 GMT', }) status, headers, body = self.call_slo(req) # oh it's *definately* been modified since then! self.assertEqual(status, '200 OK') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_last_modified_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_last_modified_json_md5) self.assertEqual(int(headers['Content-Length']), self.manifest_last_modified_slo_size) self.assertEqual(headers['Last-Modified'], 'Mon, 23 Oct 2023 10:05:32 GMT') self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/c/manifest-last-modified'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), ]) def test_if_modified_since_last_modified(self): req = swob.Request.blank( '/v1/AUTH_test/c/manifest-last-modified', headers={ 'If-Modified-Since': 'Mon, 23 Oct 2023 10:05:32 GMT', }) status, headers, body = self.call_slo(req) # nope, that was the last time it was changed self.assertEqual(status, '304 Not Modified') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_last_modified_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_last_modified_json_md5) self.assertEqual(headers['Content-Length'], '0') self.assertEqual('Mon, 23 Oct 2023 10:05:32 GMT', headers['Last-Modified']) self.assertEqual(b'', body) expected_calls = [ ('GET', '/v1/AUTH_test/c/manifest-last-modified'), ] if not self.modern_manifest_headers: # N.B. legacy manifests must refetch for accurate Etag, and then we # validate first segment before lettting swob return the error expected_calls.extend([ ('GET', '/v1/AUTH_test/c/manifest-last-modified'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ]) # we don't drain the segment's resp_iter if validation fails self.expected_unread_requests[ ('GET', '/v1/AUTH_test/gettest/a_5' '?multipart-manifest=get')] = 1 self.assertEqual(self.app.calls, expected_calls) def test_if_modified_since_now(self): now = datetime.now() last_modified = now.strftime("%a, %d %b %Y %H:%M:%S %Z") req = swob.Request.blank( '/v1/AUTH_test/c/manifest-last-modified', headers={ 'If-Modified-Since': last_modified, }) status, headers, body = self.call_slo(req) # nope, that was the last time it was changed self.assertEqual(status, '304 Not Modified') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_last_modified_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_last_modified_json_md5) self.assertEqual(headers['Content-Length'], '0') self.assertEqual('Mon, 23 Oct 2023 10:05:32 GMT', headers['Last-Modified']) self.assertEqual(b'', body) expected_calls = [ ('GET', '/v1/AUTH_test/c/manifest-last-modified'), ] if not self.modern_manifest_headers: # N.B. legacy manifests must refetch for accurate Etag, and then we # validate first segment before lettting swob return the error expected_calls.extend([ ('GET', '/v1/AUTH_test/c/manifest-last-modified'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ]) # we don't drain the segment's resp_iter if validation fails self.expected_unread_requests[ ('GET', '/v1/AUTH_test/gettest/a_5' '?multipart-manifest=get')] = 1 self.assertEqual(self.app.calls, expected_calls) def test_if_unmodified_since_ancient_date(self): req = swob.Request.blank( '/v1/AUTH_test/c/manifest-last-modified', headers={ 'If-Unmodified-Since': 'Fri, 01 Feb 2012 20:38:36 GMT', }) status, headers, body = self.call_slo(req) # oh it's *definately* been modified since then! self.assertEqual(status, '412 Precondition Failed') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_last_modified_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_last_modified_json_md5) self.assertEqual(headers['Content-Length'], '0') self.assertEqual('Mon, 23 Oct 2023 10:05:32 GMT', headers['Last-Modified']) self.assertEqual(b'', body) expected_calls = [ ('GET', '/v1/AUTH_test/c/manifest-last-modified'), ] if not self.modern_manifest_headers: # N.B. legacy manifests must refetch for accurate Etag, and then we # validate first segment before lettting swob return the error expected_calls.extend([ ('GET', '/v1/AUTH_test/c/manifest-last-modified'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ]) # we don't drain the segment's resp_iter if validation fails self.expected_unread_requests[ ('GET', '/v1/AUTH_test/gettest/a_5' '?multipart-manifest=get')] = 1 self.assertEqual(self.app.calls, expected_calls) def test_if_unmodified_since_last_modified(self): req = swob.Request.blank( '/v1/AUTH_test/c/manifest-last-modified', headers={ 'If-Unmodified-Since': 'Mon, 23 Oct 2023 10:05:32 GMT', }) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_last_modified_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_last_modified_json_md5) self.assertEqual(int(headers['Content-Length']), self.manifest_last_modified_slo_size) self.assertEqual(headers['Last-Modified'], 'Mon, 23 Oct 2023 10:05:32 GMT') self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/c/manifest-last-modified'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), ]) def test_if_unmodified_since_now(self): now = datetime.now() last_modified = now.strftime("%a, %d %b %Y %H:%M:%S %Z") req = swob.Request.blank( '/v1/AUTH_test/c/manifest-last-modified', headers={ 'If-Unmodified-Since': last_modified, }) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_last_modified_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_last_modified_json_md5) self.assertEqual(int(headers['Content-Length']), self.manifest_last_modified_slo_size) self.assertEqual(headers['Last-Modified'], 'Mon, 23 Oct 2023 10:05:32 GMT') self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/c/manifest-last-modified'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), ]) class TestSloConditionalGetNewManifest(TestSloConditionalGetOldManifest): modern_manifest_headers = True class TestPartNumber(SloGETorHEADTestCase): modern_manifest_headers = True def setUp(self): super(TestPartNumber, self).setUp() self._setup_alphabet_objects('bcdj') self._setup_manifest_bc() self._setup_manifest_abcd() self._setup_manifest_abcdefghijkl() self._setup_manifest_bc_ranges() self._setup_manifest_abcd_ranges() self._setup_manifest_abcd_subranges() self._setup_manifest_aabbccdd() self._setup_manifest_single_segment() # this b_50 object doesn't follow the alphabet convention self.app.register( 'GET', '/v1/AUTH_test/gettest/b_50', swob.HTTPPartialContent, {'Content-Length': '50', 'Etag': md5hex('b' * 50)}, 'b' * 50) self._setup_manifest_data() def test_head_part_number(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc?part-number=1', environ={'REQUEST_METHOD': 'HEAD'}) status, headers, body = self.call_slo(req) expected_calls = [ ('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=1'), ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=1') ] self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) self.assertEqual(headers['Content-Length'], '10') self.assertEqual(headers['Content-Range'], 'bytes 0-9/25') self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '2') self.assertEqual(headers['Content-Type'], 'application/octet-stream') self.assertEqual(body, b'') # it's a HEAD request, after all self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') self.assertEqual(self.app.calls, expected_calls) def test_head_part_number_refetch_path(self): # verify that any modification of the request path by a downstream # middleware is ignored when refetching req = Request.blank( '/v1/AUTH_test/gettest/mani?part-number=1', environ={'REQUEST_METHOD': 'HEAD'}) captured_calls = [] orig_call = FakeSwift.__call__ def pseudo_middleware(app, env, start_response): captured_calls.append((env['REQUEST_METHOD'], env['PATH_INFO'])) # pretend another middleware modified the path # note: for convenience, the path "modification" actually results # in one of the pre-registered paths env['PATH_INFO'] += 'fest-bc' return orig_call(app, env, start_response) with patch.object(FakeSwift, '__call__', pseudo_middleware): status, headers, body = self.call_slo(req) # pseudo-middleware gets the original path for the refetch self.assertEqual([('HEAD', '/v1/AUTH_test/gettest/mani'), ('GET', '/v1/AUTH_test/gettest/mani')], captured_calls) self.assertEqual(status, '206 Partial Content') expected_calls = [ # original path is modified... ('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=1'), # refetch: the *original* path is modified... ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=1') ] self.assertEqual(self.app.calls, expected_calls) def test_get_part_number(self): # part number 1 is b_10 req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc?part-number=1') status, headers, body = self.call_slo(req) expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=1'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get') ] self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) self.assertEqual(headers['Content-Length'], '10') self.assertEqual(headers['Content-Range'], 'bytes 0-9/25') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '2') self.assertEqual(headers['Content-Type'], 'application/octet-stream') self.assertEqual(body, b'b' * 10) self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') self.assertEqual(self.app.calls, expected_calls) # part number 2 is c_15 self.app.clear_calls() expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=2'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get') ] req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc?part-number=2') status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) self.assertEqual(headers['Content-Length'], '15') self.assertEqual(headers['Content-Range'], 'bytes 10-24/25') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '2') self.assertEqual(headers['Content-Type'], 'application/octet-stream') self.assertEqual(body, b'c' * 15) self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') self.assertEqual(self.app.calls, expected_calls) # we now test it with single segment slo self.app.clear_calls() req = Request.blank( '/v1/AUTH_test/gettest/manifest-single-segment?part-number=1') status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_single_segment_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_single_segment_json_md5) self.assertEqual(headers['Content-Length'], '50') self.assertEqual(headers['Content-Range'], 'bytes 0-49/50') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Object-Meta-Nature'], 'Regular') self.assertEqual(headers['X-Parts-Count'], '1') self.assertEqual(headers['Content-Type'], 'application/octet-stream') expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-single-segment?' 'part-number=1'), ('GET', '/v1/AUTH_test/gettest/b_50?multipart-manifest=get') ] self.assertEqual(self.app.calls, expected_calls) def test_get_part_number_sub_slo(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd?part-number=3') status, headers, body = self.call_slo(req) expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd?part-number=3'), ('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get') ] self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_abcd_json_md5) self.assertEqual(headers['Content-Length'], '20') self.assertEqual(headers['Content-Range'], 'bytes 30-49/50') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '3') self.assertEqual(headers['Content-Type'], 'application/json') self.assertEqual(body, b'd' * 20) self.assertEqual(self.app.calls, expected_calls) self.app.clear_calls() req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd?part-number=2') status, headers, body = self.call_slo(req) expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd?part-number=2'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get') ] self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_abcd_json_md5) self.assertEqual(headers['Content-Length'], '25') self.assertEqual(headers['Content-Range'], 'bytes 5-29/50') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '3') self.assertEqual(headers['Content-Type'], 'application/json') self.assertEqual(body, b'b' * 10 + b'c' * 15) self.assertEqual(self.app.calls, expected_calls) def test_get_part_number_large_manifest(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcdefghijkl?part-number=10') status, headers, body = self.call_slo(req) expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcdefghijkl?' 'part-number=10'), ('GET', '/v1/AUTH_test/gettest/j_50?multipart-manifest=get') ] self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcdefghijkl_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_abcdefghijkl_json_md5) self.assertEqual(headers['Content-Length'], '50') self.assertEqual(headers['Content-Range'], 'bytes 225-274/390') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '12') self.assertEqual(headers['Content-Type'], 'application/octet-stream') self.assertEqual(body, b'j' * 50) self.assertEqual(self.app.calls, expected_calls) def test_part_number_with_range_segments(self): req = Request.blank('/v1/AUTH_test/gettest/manifest-bc-ranges', params={'part-number': 1}) status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_ranges_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_ranges_json_md5) self.assertEqual(headers['Content-Length'], '4') self.assertEqual(headers['Content-Range'], 'bytes 0-3/%s' % self.manifest_bc_ranges_slo_size) self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '4') self.assertEqual(body, b'b' * 4) expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-bc-ranges?part-number=1'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get') ] self.assertEqual(self.app.calls, expected_calls) # since the our requested part-number is range-segment we expect Range # header on b_10 segment subrequest self.assertEqual('bytes=4-7', self.app.calls_with_headers[1].headers['Range']) def test_part_number_sub_ranges_manifest(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=3') status, headers, body = self.call_slo(req) expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd-subranges?' 'part-number=3'), ('GET', '/v1/AUTH_test/gettest/manifest-abcd-ranges'), ('GET', '/v1/AUTH_test/gettest/manifest-bc-ranges'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get') ] self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_subranges_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_abcd_subranges_json_md5) self.assertEqual(headers['Content-Length'], '5') self.assertEqual(headers['Content-Range'], 'bytes 6-10/17') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '5') self.assertEqual(headers['Content-Type'], 'application/json') self.assertEqual(body, b'c' * 2 + b'b' * 3) self.assertEqual(self.app.calls, expected_calls) def test_get_part_num_with_repeated_segments(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-aabbccdd?part-number=3', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-aabbccdd?part-number=3'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get') ] self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_aabbccdd_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_aabbccdd_json_md5) self.assertEqual(headers['Content-Length'], '10') self.assertEqual(headers['Content-Range'], 'bytes 10-19/100') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '8') self.assertEqual(headers['Content-Type'], 'application/octet-stream') self.assertEqual(body, b'b' * 10) self.assertEqual(self.app.calls, expected_calls) def test_part_number_zero_invalid(self): # part-number query param is 1-indexed, part-number=0 is no joy req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc?part-number=0') status, headers, body = self.call_slo(req) self.assertEqual(status, '400 Bad Request') self.assertNotIn('Content-Range', headers) self.assertNotIn('Etag', headers) self.assertNotIn('X-Static-Large-Object', headers) self.assertNotIn('X-Parts-Count', headers) self.assertEqual(body, b'Part number must be an integer greater than 0') expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=0') ] self.assertEqual(expected_calls, self.app.calls) self.app.clear_calls() self.slo.max_manifest_segments = 3999 req = Request.blank('/v1/AUTH_test/gettest/manifest-bc', params={'part-number': 0}) status, headers, body = self.call_slo(req) self.assertEqual(status, '400 Bad Request') self.assertNotIn('Content-Range', headers) self.assertNotIn('Etag', headers) self.assertNotIn('X-Static-Large-Object', headers) self.assertNotIn('X-Parts-Count', headers) self.assertEqual(body, b'Part number must be an integer greater than 0') self.assertEqual(expected_calls, self.app.calls) def test_head_part_number_zero_invalid(self): # you can HEAD part-number=0 either req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc', method='HEAD', params={'part-number': 0}) status, headers, body = self.call_slo(req) self.assertEqual(status, '400 Bad Request') self.assertNotIn('Content-Range', headers) self.assertNotIn('Etag', headers) self.assertNotIn('X-Static-Large-Object', headers) self.assertNotIn('X-Parts-Count', headers) self.assertEqual(body, b'') # HEAD response, makes sense expected_calls = [ ('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=0') ] self.assertEqual(expected_calls, self.app.calls) def test_part_number_zero_invalid_on_subrange(self): # either manifest, doesn't matter, part-number=0 is always invalid req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=0') status, headers, body = self.call_slo(req) self.assertEqual(status, '400 Bad Request') self.assertNotIn('Content-Range', headers) self.assertNotIn('Etag', headers) self.assertNotIn('X-Static-Large-Object', headers) self.assertNotIn('X-Parts-Count', headers) self.assertEqual(body, b'Part number must be an integer greater than 0') expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=0') ] self.assertEqual(expected_calls, self.app.calls) def test_negative_part_number_invalid(self): # negative numbers are never any good req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc?part-number=-1') status, headers, body = self.call_slo(req) self.assertEqual(status, '400 Bad Request') self.assertNotIn('Content-Range', headers) self.assertNotIn('Etag', headers) self.assertNotIn('X-Static-Large-Object', headers) self.assertNotIn('X-Parts-Count', headers) self.assertEqual(body, b'Part number must be an integer greater than 0') expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=-1') ] self.assertEqual(expected_calls, self.app.calls) def test_head_negative_part_number_invalid_on_subrange(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd-subranges', method='HEAD', params={'part-number': '-1'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '400 Bad Request') self.assertNotIn('Content-Range', headers) self.assertNotIn('Etag', headers) self.assertNotIn('X-Static-Large-Object', headers) self.assertNotIn('X-Parts-Count', headers) self.assertEqual(body, b'') expected_calls = [ ('HEAD', '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=-1') ] self.assertEqual(expected_calls, self.app.calls) def test_head_non_integer_part_number_invalid(self): # some kind of string is bad too req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc', method='HEAD', params={'part-number': 'foo'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '400 Bad Request') self.assertEqual(body, b'') expected_calls = [ ('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=foo') ] self.assertEqual(expected_calls, self.app.calls) def test_get_non_integer_part_number_invalid(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc', params={'part-number': 'foo'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '400 Bad Request') self.assertNotIn('Content-Range', headers) self.assertNotIn('Etag', headers) self.assertNotIn('X-Static-Large-Object', headers) self.assertNotIn('X-Parts-Count', headers) self.assertEqual(body, b'Part number must be an integer greater' b' than 0') expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=foo') ] self.assertEqual(expected_calls, self.app.calls) def test_get_out_of_range_part_number(self): # you can't go past the actual number of parts either req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc?part-number=4') status, headers, body = self.call_slo(req) self.assertEqual(status, '416 Requested Range Not Satisfiable') self.assertEqual(headers['Content-Range'], 'bytes */%d' % self.manifest_bc_slo_size) self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '2') self.assertEqual(int(headers['Content-Length']), len(body)) self.assertEqual(body, b'The requested part number is not ' b'satisfiable') self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') expected_app_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=4'), ] self.assertEqual(self.app.calls, expected_app_calls) self.app.clear_calls() req = Request.blank( '/v1/AUTH_test/gettest/manifest-single-segment?part-number=2') status, headers, body = self.call_slo(req) self.assertEqual(status, '416 Requested Range Not Satisfiable') self.assertEqual(headers['Content-Range'], 'bytes */%d' % self.manifest_single_segment_slo_size) self.assertEqual(int(headers['Content-Length']), len(body)) self.assertEqual(headers['Etag'], '"%s"' % self.manifest_single_segment_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_single_segment_json_md5) self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '1') self.assertEqual(body, b'The requested part number is not ' b'satisfiable') self.assertEqual(headers['X-Object-Meta-Nature'], 'Regular') expected_app_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-single-segment?' 'part-number=2'), ] self.assertEqual(self.app.calls, expected_app_calls) def test_head_out_of_range_part_number(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc?part-number=4') req.method = 'HEAD' status, headers, body = self.call_slo(req) self.assertEqual(status, '416 Requested Range Not Satisfiable') self.assertEqual(headers['Content-Range'], 'bytes */%d' % self.manifest_bc_slo_size) self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '2') self.assertEqual(int(headers['Content-Length']), len(body)) self.assertEqual(body, b'') self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') expected_app_calls = [ ('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=4'), # segments needed ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=4') ] self.assertEqual(self.app.calls, expected_app_calls) def test_part_number_exceeds_max_manifest_segments_is_ok(self): # verify that an existing part can be fetched regardless of the current # max_manifest_segments req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc?part-number=2') self.slo.max_manifest_segments = 1 status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) self.assertEqual(headers['Content-Length'], '15') self.assertEqual(headers['Content-Range'], 'bytes 10-24/25') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '2') self.assertEqual(headers['Content-Type'], 'application/octet-stream') self.assertEqual(body, b'c' * 15) expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=2'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get') ] self.assertEqual(self.app.calls, expected_calls) def test_part_number_ignored_for_non_slo_object(self): # verify that a part-number param is ignored for a non-slo object def do_test(query_string): self.app.clear_calls() req = Request.blank( '/v1/AUTH_test/gettest/c_15?%s' % query_string) self.slo.max_manifest_segments = 1 status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(headers['Etag'], '%s' % md5hex('c' * 15)) self.assertEqual(headers['Content-Length'], '15') self.assertEqual(body, b'c' * 15) self.assertEqual(1, self.app.call_count) method, path = self.app.calls[0] actual_req = Request.blank(path, method=method) self.assertEqual(req.path, actual_req.path) self.assertEqual(req.params, actual_req.params) do_test('part-number=-1') do_test('part-number=0') do_test('part-number=1') do_test('part-number=2') do_test('part-number=foo') do_test('part-number=foo&multipart-manifest=get') def test_part_number_ignored_for_non_slo_object_with_range(self): # verify that a part-number param is ignored for a non-slo object def do_test(query_string): self.app.clear_calls() req = Request.blank( '/v1/AUTH_test/gettest/c_15?%s' % query_string, headers={'Range': 'bytes=1-2'}) self.slo.max_manifest_segments = 1 status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '%s' % md5hex('c' * 15)) self.assertEqual(headers['Content-Length'], '2') self.assertEqual(headers['Content-Range'], 'bytes 1-2/15') self.assertEqual(body, b'c' * 2) self.assertEqual(1, self.app.call_count) method, path = self.app.calls[0] actual_req = Request.blank(path, method=method) self.assertEqual(req.path, actual_req.path) self.assertEqual(req.params, actual_req.params) do_test('part-number=-1') do_test('part-number=0') do_test('part-number=1') do_test('part-number=2') do_test('part-number=foo') do_test('part-number=foo&multipart-manifest=get') def test_part_number_ignored_for_manifest_get(self): def do_test(query_string): self.app.clear_calls() req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc?%s' % query_string) self.slo.max_manifest_segments = 1 status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(headers['Etag'], self.manifest_bc_json_md5) self.assertEqual(headers['Content-Length'], str(self.manifest_bc_json_size)) self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['Content-Type'], 'application/json; charset=utf-8') self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') self.assertEqual(1, self.app.call_count) method, path = self.app.calls[0] actual_req = Request.blank(path, method=method) self.assertEqual(req.path, actual_req.path) self.assertEqual(req.params, actual_req.params) do_test('part-number=-1&multipart-manifest=get') do_test('part-number=0&multipart-manifest=get') do_test('part-number=1&multipart-manifest=get') do_test('part-number=2&multipart-manifest=get') do_test('part-number=foo&multipart-manifest=get') def test_head_out_of_range_part_number_on_subrange(self): # you can't go past the actual number of parts either req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd-subranges', method='HEAD', params={'part-number': 6}) expected_calls = [ ('HEAD', '/v1/AUTH_test/gettest/manifest-abcd-subranges?' 'part-number=6'), ('GET', '/v1/AUTH_test/gettest/manifest-abcd-subranges?' 'part-number=6')] status, headers, body = self.call_slo(req) self.assertEqual(status, '416 Requested Range Not Satisfiable') self.assertEqual(headers['Content-Range'], 'bytes */%d' % self.manifest_abcd_subranges_slo_size) self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_subranges_slo_etag) self.assertEqual(headers['X-Manifest-Etag'], self.manifest_abcd_subranges_json_md5) self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '5') self.assertEqual(int(headers['Content-Length']), len(body)) self.assertEqual(body, b'') self.assertEqual(self.app.calls, expected_calls) def test_range_with_part_number_is_error(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=2', headers={'Range': 'bytes=4-12'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '400 Bad Request') self.assertNotIn('Content-Range', headers) self.assertNotIn('Etag', headers) self.assertNotIn('X-Static-Large-Object', headers) self.assertNotIn('X-Parts-Count', headers) self.assertEqual(body, b'Range requests are not supported with ' b'part number queries') expected_calls = [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=2') ] self.assertEqual(expected_calls, self.app.calls) def test_head_part_number_subrange(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd-subranges', method='HEAD', params={'part-number': 2}) status, headers, body = self.call_slo(req) # Range header can be ignored in a HEAD request self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_subranges_slo_etag) self.assertEqual(headers['Content-Length'], '1') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['Content-Type'], 'application/json') self.assertEqual(headers['X-Parts-Count'], '5') self.assertEqual(body, b'') # it's a HEAD request, after all expected_calls = [ ('HEAD', '/v1/AUTH_test/gettest/manifest-abcd-subranges' '?part-number=2'), ('GET', '/v1/AUTH_test/gettest/manifest-abcd-subranges' '?part-number=2'), ] self.assertEqual(self.app.calls, expected_calls) def test_head_part_number_data_manifest(self): req = Request.blank( '/v1/AUTH_test/c/manifest-data', method='HEAD', params={'part-number': 1}) status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_data_slo_etag) self.assertEqual(headers['Content-Length'], '6') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '3') self.assertEqual(body, b'') # it's a HEAD request, after all expected_calls = [ ('HEAD', '/v1/AUTH_test/c/manifest-data?part-number=1'), ('GET', '/v1/AUTH_test/c/manifest-data?part-number=1'), ] self.assertEqual(self.app.calls, expected_calls) def test_get_part_number_data_manifest(self): req = Request.blank( '/v1/AUTH_test/c/manifest-data', params={'part-number': 3}) status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_data_slo_etag) self.assertEqual(headers['Content-Length'], '6') self.assertEqual(headers['X-Static-Large-Object'], 'true') self.assertEqual(headers['X-Parts-Count'], '3') self.assertEqual(body, b'ABCDEF') expected_calls = [ ('GET', '/v1/AUTH_test/c/manifest-data?part-number=3'), ] self.assertEqual(self.app.calls, expected_calls) class TestPartNumberLegacyManifest(TestPartNumber): modern_manifest_headers = False class TestSloBulkDeleter(unittest.TestCase): def test_reused_logger(self): slo_mware = slo.filter_factory({})('fake app') self.assertTrue(slo_mware.logger is slo_mware.bulk_deleter.logger) def test_passes_through_concurrency(self): slo_mware = slo.filter_factory({'delete_concurrency': 5})('fake app') self.assertEqual(5, slo_mware.bulk_deleter.delete_concurrency) def test_uses_big_max_deletes(self): slo_mware = slo.filter_factory( {'max_manifest_segments': 123456789})('fake app') self.assertGreaterEqual( slo_mware.bulk_deleter.max_deletes_per_request, 123456789) class TestSwiftInfo(unittest.TestCase): def setUp(self): registry._swift_info = {} registry._swift_admin_info = {} def test_registered_defaults(self): mware = slo.filter_factory({})('have to pass in an app') swift_info = registry.get_swift_info() self.assertTrue('slo' in swift_info) self.assertEqual(swift_info['slo'].get('max_manifest_segments'), mware.max_manifest_segments) self.assertEqual(swift_info['slo'].get('min_segment_size'), 1) self.assertEqual(swift_info['slo'].get('max_manifest_size'), mware.max_manifest_size) self.assertIs(swift_info['slo'].get('allow_async_delete'), True) self.assertEqual(1000, mware.max_manifest_segments) self.assertEqual(8388608, mware.max_manifest_size) self.assertEqual(1048576, mware.rate_limit_under_size) self.assertEqual(10, mware.rate_limit_after_segment) self.assertEqual(1, mware.rate_limit_segments_per_sec) self.assertEqual(10, mware.yield_frequency) self.assertEqual(2, mware.concurrency) self.assertEqual(2, mware.bulk_deleter.delete_concurrency) self.assertIs(True, mware.allow_async_delete) def test_registered_non_defaults(self): conf = dict( max_manifest_segments=500, max_manifest_size=1048576, rate_limit_under_size=2097152, rate_limit_after_segment=20, rate_limit_segments_per_sec=2, yield_frequency=5, concurrency=1, delete_concurrency=3, allow_async_delete='n') mware = slo.filter_factory(conf)('have to pass in an app') swift_info = registry.get_swift_info() self.assertTrue('slo' in swift_info) self.assertEqual(swift_info['slo'].get('max_manifest_segments'), 500) self.assertEqual(swift_info['slo'].get('min_segment_size'), 1) self.assertEqual(swift_info['slo'].get('max_manifest_size'), 1048576) self.assertIs(swift_info['slo'].get('allow_async_delete'), False) self.assertEqual(500, mware.max_manifest_segments) self.assertEqual(1048576, mware.max_manifest_size) self.assertEqual(2097152, mware.rate_limit_under_size) self.assertEqual(20, mware.rate_limit_after_segment) self.assertEqual(2, mware.rate_limit_segments_per_sec) self.assertEqual(5, mware.yield_frequency) self.assertEqual(1, mware.concurrency) self.assertEqual(3, mware.bulk_deleter.delete_concurrency) self.assertIs(False, mware.allow_async_delete) class TestNonSloPassthrough(SloGETorHEADTestCase): def setUp(self): super(TestNonSloPassthrough, self).setUp() self._setup_alphabet_objects('a') body = b'big' * 1000 self.app.register( 'GET', '/v1/AUTH_test/rangetest/big', swob.HTTPOk, { 'Content-Length': len(body), 'Etag': md5hex(body), }, body=body) def test_get_nonmanifest_passthrough(self): req = Request.blank( '/v1/AUTH_test/gettest/a_5', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(headers['Etag'], md5hex('a' * 5)) self.assertEqual(headers['Content-Length'], '5') self.assertNotIn('X-Static-Large-Object', headers) self.assertNotIn('X-Manifest-Etag', headers) self.assertEqual(body, b'aaaaa') self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/gettest/a_5'), ]) def test_non_slo_range_passthrough(self): req = Request.blank( '/v1/AUTH_test/rangetest/big', headers={'Range': 'bytes=0-4'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Content-Length'], '5') self.assertEqual(headers['Content-Range'], 'bytes 0-4/3000') self.assertEqual(body, b'bigbi') self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/rangetest/big'), ]) def test_non_slo_range_unsatisfiable_passthrough(self): req = Request.blank( '/v1/AUTH_test/rangetest/big', headers={'Range': 'bytes=3001-'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '416 Requested Range Not Satisfiable') self.assertEqual(headers['Content-Range'], 'bytes */3000') self.assertIn(b'Requested Range Not Satisfiable', body) self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/rangetest/big'), ]) def test_non_slo_multi_range_passthrough(self): req = Request.blank( '/v1/AUTH_test/rangetest/big', headers={'Range': 'bytes=1-2,3-4'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertNotIn('Content-Range', headers) ct, params = parse_content_type(headers['Content-Type']) self.assertEqual(ct, 'multipart/byteranges') params = dict(params) boundary = params.get('boundary') if six.PY3: boundary = boundary.encode('utf-8') self.assertEqual(len(body), int(headers['Content-Length'])) got_mime_docs = [] for mime_doc_fh in iter_multipart_mime_documents( BytesIO(body), boundary): headers = parse_mime_headers(mime_doc_fh) body = mime_doc_fh.read() got_mime_docs.append((headers, body)) self.assertEqual(len(got_mime_docs), 2) first_range_headers, first_range_body = got_mime_docs[0] self.assertEqual(first_range_headers['Content-Range'], 'bytes 1-2/3000') self.assertEqual(first_range_body, b'ig') second_range_headers, second_range_body = got_mime_docs[1] self.assertEqual(second_range_headers['Content-Range'], 'bytes 3-4/3000') # 012 34 5678 # big bi gbig self.assertEqual(second_range_body, b'bi') self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/rangetest/big'), ]) def test_non_slo_multi_range_partially_satisfiable_passthrough(self): req = Request.blank( '/v1/AUTH_test/rangetest/big', headers={'Range': 'bytes=1-2,3-4,3001-'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertNotIn('Content-Range', headers) self.assertEqual(int(headers['Content-Length']), len(body)) ct, params = parse_content_type(headers['Content-Type']) self.assertEqual(ct, 'multipart/byteranges') params = dict(params) boundary = params.get('boundary') if six.PY3: boundary = boundary.encode('utf-8') self.assertEqual(len(body), int(headers['Content-Length'])) got_mime_docs = [] for mime_doc_fh in iter_multipart_mime_documents( BytesIO(body), boundary): headers = parse_mime_headers(mime_doc_fh) body = mime_doc_fh.read() got_mime_docs.append((headers, body)) self.assertEqual(len(got_mime_docs), 2) first_range_headers, first_range_body = got_mime_docs[0] self.assertEqual(first_range_headers['Content-Range'], 'bytes 1-2/3000') self.assertEqual(first_range_body, b'ig') second_range_headers, second_range_body = got_mime_docs[1] self.assertEqual(second_range_headers['Content-Range'], 'bytes 3-4/3000') self.assertEqual(second_range_body, b'bi') self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/rangetest/big'), ]) def test_non_slo_multi_range_unsatisfiable_passthrough(self): req = Request.blank( '/v1/AUTH_test/rangetest/big', headers={'Range': 'bytes=3001-,3005-3010'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '416 Requested Range Not Satisfiable') self.assertEqual(headers['Content-Range'], 'bytes */3000') self.assertEqual(int(headers['Content-Length']), len(body)) self.assertIn(b'Requested Range Not Satisfiable', body) self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/rangetest/big'), ]) def test_non_slo_multi_range_starting_beyond_multipart_resp_length(self): req = Request.blank( '/v1/AUTH_test/rangetest/big', headers={'Range': 'bytes=1000-1002,2000-2002'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '206 Partial Content') self.assertNotIn('Content-Range', headers) self.assertEqual(int(headers['Content-Length']), len(body)) ct, params = parse_content_type(headers['Content-Type']) self.assertEqual(ct, 'multipart/byteranges') params = dict(params) boundary = params.get('boundary') if six.PY3: boundary = boundary.encode('utf-8') self.assertEqual(len(body), int(headers['Content-Length'])) got_mime_docs = [] for mime_doc_fh in iter_multipart_mime_documents( BytesIO(body), boundary): headers = parse_mime_headers(mime_doc_fh) body = mime_doc_fh.read() got_mime_docs.append((headers, body)) self.assertEqual(len(got_mime_docs), 2) first_range_headers, first_range_body = got_mime_docs[0] self.assertEqual(first_range_headers['Content-Range'], 'bytes 1000-1002/3000') self.assertEqual(first_range_body, b'igb') second_range_headers, second_range_body = got_mime_docs[1] self.assertEqual(second_range_headers['Content-Range'], 'bytes 2000-2002/3000') self.assertEqual(second_range_body, b'gbi') self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/rangetest/big'), ]) class TestRespAttrs(unittest.TestCase): def test_init_calculates_is_legacy(self): attrs = slo.RespAttrs(True, 123456789.12345, 'manifest-etag', 'slo-etag', 999) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertIsInstance(attrs.timestamp, Timestamp) self.assertEqual('manifest-etag', attrs.json_md5) self.assertEqual('slo-etag', attrs.slo_etag) self.assertEqual(999, attrs.slo_size) # we gave it etag and size! self.assertTrue(attrs._has_size_and_etag()) self.assertFalse(attrs.is_legacy) def test_init_converts_timestamps_from_strings(self): attrs = slo.RespAttrs(True, '123456789.12345', 'manifest-etag', 'slo-etag', 999) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertIsInstance(attrs.timestamp, Timestamp) self.assertEqual('manifest-etag', attrs.json_md5) self.assertEqual('slo-etag', attrs.slo_etag) self.assertEqual(999, attrs.slo_size) # we gave it etag and size! self.assertTrue(attrs._has_size_and_etag()) self.assertFalse(attrs.is_legacy) def test_default_types(self): attrs = slo.RespAttrs(None, None, None, None, None) # types are correct, values are default/place-holders self.assertTrue(attrs.is_slo is False) # not None! self.assertEqual(0, attrs.timestamp) self.assertIsInstance(attrs.timestamp, Timestamp) self.assertEqual('', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) # we didn't provide etag & size self.assertFalse(attrs._has_size_and_etag()) self.assertTrue(attrs.is_legacy) def test_init_with_no_sysmeta(self): now = Timestamp.now() attrs = slo.RespAttrs(True, now.normal, None, None, None) self.assertTrue(attrs.is_slo) self.assertEqual(now, attrs.timestamp) self.assertIsInstance(attrs.timestamp, Timestamp) self.assertEqual('', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) # we didn't provide etag & size self.assertFalse(attrs._has_size_and_etag()) self.assertTrue(attrs.is_legacy) def test_init_with_no_sysmeta_offset(self): now = Timestamp.now(offset=123) attrs = slo.RespAttrs(True, now.internal, None, None, None) self.assertTrue(attrs.is_slo) self.assertEqual(now, attrs.timestamp) self.assertIsInstance(attrs.timestamp, Timestamp) self.assertEqual('', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) # we didn't provide etag & size self.assertFalse(attrs._has_size_and_etag()) self.assertTrue(attrs.is_legacy) def test_from_empty_headers(self): attrs = slo.RespAttrs.from_headers([]) self.assertFalse(attrs.is_slo) self.assertEqual(0, attrs.timestamp) self.assertEqual('', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) self.assertTrue(attrs.is_legacy) def test_from_only_timestamp(self): now = Timestamp.now(offset=1) attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', now.internal), ('X-Irrelevant', 'ignored')]) self.assertFalse(attrs.is_slo) self.assertEqual(now, attrs.timestamp) self.assertEqual('', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) self.assertTrue(attrs.is_legacy) def test_legacy_slo_sysmeta(self): attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', '123456789.12345'), ('Etag', 'manifest-etag'), ('X-Static-lARGE-Object', 'yes')]) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual('manifest-etag', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) self.assertTrue(attrs.is_legacy) def test_partial_modern_sysmeta(self): # missing slo etag attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', '123456789.12345'), ('Etag', 'manifest-etag'), ('X-Static-lARGE-Object', 'yes'), ('x-object-sysmeta-slo-size', '1234')]) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual('manifest-etag', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(1234, attrs.slo_size) self.assertTrue(attrs.is_legacy) # missing slo size attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', '123456789.12345'), ('Etag', 'manifest-etag'), ('X-Static-lARGE-Object', 'yes'), ('x-object-sysmeta-slo-etag', 'slo-etag')]) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual('manifest-etag', attrs.json_md5) self.assertEqual('slo-etag', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) self.assertTrue(attrs.is_legacy) # missing manifest etag attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', '123456789.12345'), ('X-Static-lARGE-Object', 'yes'), ('x-object-sysmeta-slo-size', '1234'), ('x-object-sysmeta-slo-etag', 'slo-etag')]) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual('', attrs.json_md5) self.assertEqual('slo-etag', attrs.slo_etag) self.assertEqual(1234, attrs.slo_size) # missing Etag might be some kind of bug, but it has all sysmeta self.assertFalse(attrs.is_legacy) def test_invalid_sysmeta(self): attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', '123456789.12345'), ('X-Static-lARGE-Object', 'yes'), ('x-object-sysmeta-slo-size', 'huge!')]) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual('', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) self.assertTrue(attrs.is_legacy) attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', '123456789.12345'), ('X-Static-lARGE-Object', 'yes'), ('e-TAG', 'wrong!'), ('x-object-sysmeta-slo-size', '')]) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual('', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) self.assertTrue(attrs.is_legacy) def test_from_valid_sysmeta(self): attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', '123456789.12345'), ('Etag', 'manifest-etag'), ('X-Static-lARGE-Object', 'yes'), ('x-object-sysmeta-slo-etag', 'slo-tag'), ('x-object-sysmeta-slo-size', '1234')]) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual('manifest-etag', attrs.json_md5) self.assertEqual('slo-tag', attrs.slo_etag) self.assertEqual(1234, attrs.slo_size) self.assertFalse(attrs.is_legacy) def test_from_regular_object(self): now = Timestamp.now() attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', now.normal), ('Etag', 'object-etag')]) self.assertFalse(attrs.is_slo) self.assertEqual(now, attrs.timestamp) # N.B. we only set manifest_etag on slo objects self.assertEqual('', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) self.assertTrue(attrs.is_legacy) def test_non_slo_with_sysmeta(self): attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', '123456789.12345'), ('X-Static-lARGE-Object', 'false')]) self.assertFalse(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual('', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) self.assertTrue(attrs.is_legacy) attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', '123456789.12345'), ('Etag', 'segment-etag'), ('x-object-sysmeta-slo-etag', 'tag'), ('x-object-sysmeta-slo-size', '1234')]) # this is NOT an SLO self.assertFalse(attrs.is_slo) self.assertEqual('', attrs.json_md5) # ... but we set these based on the sysmeta values self.assertEqual('tag', attrs.slo_etag) self.assertEqual(1234, attrs.slo_size) self.assertEqual(123456789.12345, attrs.timestamp) # I hope someday a non-slo with slo sysmeta *will* be just a legacy, # see lp bug #2035158 self.assertFalse(attrs.is_legacy) def _legacy_from_headers(self): attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', '123456789.12345'), ('Etag', 'manifest-etag'), ('X-Static-lARGE-Object', 'yes')]) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual('manifest-etag', attrs.json_md5) self.assertEqual('', attrs.slo_etag) self.assertEqual(-1, attrs.slo_size) self.assertTrue(attrs.is_legacy) return attrs def test_update_from_segments(self): attrs = self._legacy_from_headers() segments = [ {'hash': 'abc', 'bytes': 2}, {'hash': 'def', 'bytes': 3}, ] slo._annotate_segments(segments) attrs.update_from_segments(segments) exp_etag = md5('abcdef'.encode('ascii'), usedforsecurity=False) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual(exp_etag.hexdigest(), attrs.slo_etag) self.assertEqual(5, attrs.slo_size) # N.B. it's still a legacy manifest self.assertTrue(attrs.is_legacy) def test_update_from_segments_with_raw_data(self): attrs = self._legacy_from_headers() raw_data = b'something' segments = [ {'hash': 'abc', 'bytes': 2}, {'data': base64.b64encode(raw_data)}, ] slo._annotate_segments(segments) attrs.update_from_segments(segments) raw_data_checksum = md5(raw_data).hexdigest() exp_etag = md5(('abc' + raw_data_checksum).encode('ascii'), usedforsecurity=False) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual(exp_etag.hexdigest(), attrs.slo_etag) self.assertEqual(11, attrs.slo_size) # N.B. it's still a legacy manifest self.assertTrue(attrs.is_legacy) def test_update_from_segments_with_range(self): attrs = self._legacy_from_headers() segments = [ {'hash': 'abc', 'bytes': 2}, {'hash': 'def', 'range': '1-2'}, ] slo._annotate_segments(segments) attrs.update_from_segments(segments) exp_etag = md5('abcdef:1-2;'.encode('ascii'), usedforsecurity=False) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual(exp_etag.hexdigest(), attrs.slo_etag) self.assertEqual(4, attrs.slo_size) # N.B. it's still a legacy manifest self.assertTrue(attrs.is_legacy) def test_update_from_segments_with_sub_slo(self): attrs = self._legacy_from_headers() content_type = 'application/octet-stream' content_type += ";swift_bytes=%d" % 5 segments = [ {'hash': 'abc', 'bytes': 2}, {'hash': '123', 'sub_slo': True, 'content_type': content_type}, ] slo._annotate_segments(segments) attrs.update_from_segments(segments) exp_etag = md5('abc123'.encode('ascii'), usedforsecurity=False) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual(exp_etag.hexdigest(), attrs.slo_etag) self.assertEqual(7, attrs.slo_size) # N.B. it's still a legacy manifest self.assertTrue(attrs.is_legacy) def test_update_from_segments_with_sub_slo_range(self): attrs = self._legacy_from_headers() content_type = 'application/octet-stream' content_type += ";swift_bytes=%d" % 5 segments = [ {'hash': 'abc', 'bytes': 2}, {'hash': '123', 'sub_slo': True, 'content_type': content_type, 'range': '2-4'}, ] slo._annotate_segments(segments) attrs.update_from_segments(segments) exp_etag = md5('abc123:2-4;'.encode('ascii'), usedforsecurity=False) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual(exp_etag.hexdigest(), attrs.slo_etag) self.assertEqual(5, attrs.slo_size) # N.B. it's still a legacy manifest self.assertTrue(attrs.is_legacy) def test_update_from_segments_not_legacy(self): attrs = slo.RespAttrs.from_headers( [('X-Backend-Timestamp', '123456789.12345'), ('X-Static-lARGE-Object', 'yes'), ('x-object-sysmeta-slo-etag', 'tag'), ('x-object-sysmeta-slo-size', '1234')]) segments = 'not even json; does not matter' attrs.update_from_segments(segments) self.assertTrue(attrs.is_slo) self.assertEqual(123456789.12345, attrs.timestamp) self.assertEqual('tag', attrs.slo_etag) self.assertEqual(1234, attrs.slo_size) # N.B. it's still a legacy manifest self.assertFalse(attrs.is_legacy) if __name__ == '__main__': unittest.main()