x/swift3
Code Issues Proposed changes

Support x-amz-copy-source-range for Upload Part - Copy requests

Browse Source 
Previously, we only supported copying complete objects for multipart
uploads. But according to AWS's docs [1], you may optionally specify a
single range to copy.

[1] http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html

Change-Id: I8f77a401ca896265dfb4b9ae714f7761554f7ea7
This commit is contained in:
Tim Burke 2015-12-08 17:04:36 -08:00
parent 32c7ea5e70
commit c8ef414ce5
7 changed files with 186 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
swift3/controllers/multi_upload.py
View File

@ -46,6 +46,7 @@ import os
import re
import sys
from swift.common.swob import Range
from swift.common.utils import json
from swift.common.db import utf8encode
@ -122,7 +123,33 @@ class PartController(Controller):
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
req.check_copy_source(self.app)
source_resp = req.check_copy_source(self.app)
if 'X-Amz-Copy-Source' in req.headers and \
'X-Amz-Copy-Source-Range' in req.headers:
rng = req.headers['X-Amz-Copy-Source-Range']
header_valid = True
try:
rng_obj = Range(rng)
if len(rng_obj.ranges) != 1:
header_valid = False
except ValueError:
header_valid = False
if not header_valid:
err_msg = ('The x-amz-copy-source-range value must be of the '
'form bytes=first-last where first and last are '
'the zero-based offsets of the first and last '
'bytes to copy')
raise InvalidArgument('x-amz-source-range', rng, err_msg)
source_size = int(source_resp.headers['Content-Length'])
if not rng_obj.ranges_for_length(source_size):
err_msg = ('Range specified is not valid for source object '
'of size: %s' % source_size)
raise InvalidArgument('x-amz-source-range', rng, err_msg)
req.headers['Range'] = rng
del req.headers['X-Amz-Copy-Source-Range']
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:

8
swift3/controllers/obj.py
View File

@ -20,7 +20,8 @@ from swift.common.swob import Range, content_range_header_value
from swift3.utils import S3Timestamp
from swift3.controllers.base import Controller
from swift3.response import S3NotImplemented, InvalidRange, NoSuchKey
from swift3.response import S3NotImplemented, InvalidRange, NoSuchKey, \
InvalidArgument
class ObjectController(Controller):
@ -99,6 +100,11 @@ class ObjectController(Controller):
# set X-Timestamp by swift3 to use at copy resp body
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
if all(h in req.headers
for h in ('X-Amz-Copy-Source', 'X-Amz-Copy-Source-Range')):
raise InvalidArgument('x-amz-copy-source-range',
req.headers['X-Amz-Copy-Source-Range'],
'Illegal copy header')
req.check_copy_source(self.app)
resp = req.get_response(self.app)

57
swift3/request.py
View File

@ -326,36 +326,41 @@ class Request(swob.Request):
"""
check_copy_source checks the copy source existence and if copying an
object to itself, for illegal request parameters
:returns: the source HEAD response
"""
if 'X-Amz-Copy-Source' in self.headers:
src_path = unquote(self.headers['X-Amz-Copy-Source'])
src_path = src_path if src_path.startswith('/') else \
('/' + src_path)
src_bucket, src_obj = split_path(src_path, 0, 2, True)
headers = swob.HeaderKeyDict()
headers.update(self._copy_source_headers())
if 'X-Amz-Copy-Source' not in self.headers:
return None
src_resp = self.get_response(app, 'HEAD', src_bucket, src_obj,
headers=headers)
if src_resp.status_int == 304: # pylint: disable-msg=E1101
raise PreconditionFailed()
src_path = unquote(self.headers['X-Amz-Copy-Source'])
src_path = src_path if src_path.startswith('/') else \
('/' + src_path)
src_bucket, src_obj = split_path(src_path, 0, 2, True)
headers = swob.HeaderKeyDict()
headers.update(self._copy_source_headers())
self.headers['X-Amz-Copy-Source'] = \
'/' + self.headers['X-Amz-Copy-Source'].lstrip('/')
source_container, source_obj = \
split_path(self.headers['X-Amz-Copy-Source'], 1, 2, True)
src_resp = self.get_response(app, 'HEAD', src_bucket, src_obj,
headers=headers)
if src_resp.status_int == 304: # pylint: disable-msg=E1101
raise PreconditionFailed()
if (self.container_name == source_container and
self.object_name == source_obj):
if self.headers.get('x-amz-metadata-directive',
'COPY') == 'COPY':
raise InvalidRequest("This copy request is illegal "
"because it is trying to copy an "
"object to itself without "
"changing the object's metadata, "
"storage class, website redirect "
"location or encryption "
"attributes.")
self.headers['X-Amz-Copy-Source'] = \
'/' + self.headers['X-Amz-Copy-Source'].lstrip('/')
source_container, source_obj = \
split_path(self.headers['X-Amz-Copy-Source'], 1, 2, True)
if (self.container_name == source_container and
self.object_name == source_obj and
self.headers.get('x-amz-metadata-directive',
'COPY') == 'COPY'):
raise InvalidRequest("This copy request is illegal "
"because it is trying to copy an "
"object to itself without "
"changing the object's metadata, "
"storage class, website redirect "
"location or encryption "
"attributes.")
return src_resp
def _canonical_uri(self):
raw_path_info = self.environ.get('RAW_PATH_INFO', self.path)

65
swift3/test/functional/test_multi_upload.py
View File

@ -57,13 +57,16 @@ class TestSwift3MultiUpload(Swift3FunctionalTestCase):
return status, headers, body
def _upload_part_copy(self, src_bucket, src_obj, dst_bucket, dst_key,
upload_id, part_num=1):
upload_id, part_num=1, src_range=None):
src_path = '%s/%s' % (src_bucket, src_obj)
query = 'partNumber=%s&uploadId=%s' % (part_num, upload_id)
req_headers = {'X-Amz-Copy-Source': src_path}
if src_range:
req_headers['X-Amz-Copy-Source-Range'] = src_range
status, headers, body = \
self.conn.make_request('PUT', dst_bucket, dst_key,
headers={'X-Amz-Copy-Source': src_path},
headers=req_headers,
query=query)
elem = fromstring(body, 'CopyPartResult')
etag = elem.find('ETag').text.strip('"')
@ -179,8 +182,39 @@ class TestSwift3MultiUpload(Swift3FunctionalTestCase):
self.assertTrue('etag' not in headers)
elem = fromstring(body, 'CopyPartResult')
last_modified = elem.find('LastModified').text
self.assertTrue(last_modified is not None)
last_modified_1 = elem.find('LastModified').text
self.assertTrue(last_modified_1 is not None)
self.assertEquals(resp_etag, etag)
# Upload Part Copy Range
key, upload_id = uploads[1]
src_bucket = 'bucket2'
src_obj = 'obj4'
src_content = 'y' * (MIN_SEGMENT_SIZE / 2) + 'z' * MIN_SEGMENT_SIZE
src_range = 'bytes=0-%d' % (MIN_SEGMENT_SIZE - 1)
etag = md5(src_content[:MIN_SEGMENT_SIZE]).hexdigest()
# prepare src obj
self.conn.make_request('PUT', src_bucket)
self.conn.make_request('PUT', src_bucket, src_obj, body=src_content)
_, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj)
self.assertCommonResponseHeaders(headers)
status, headers, body, resp_etag = \
self._upload_part_copy(src_bucket, src_obj, bucket,
key, upload_id, 2, src_range)
self.assertEquals(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEquals(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEquals(headers['content-length'], str(len(body)))
self.assertTrue('etag' not in headers)
elem = fromstring(body, 'CopyPartResult')
last_modified_2 = elem.find('LastModified').text
self.assertTrue(last_modified_2 is not None)
self.assertEquals(resp_etag, etag)
@ -193,10 +227,18 @@ class TestSwift3MultiUpload(Swift3FunctionalTestCase):
elem = fromstring(body, 'ListPartsResult')
# FIXME: COPY result drops mili/microseconds but GET doesn't
last_modified_get = elem.find('Part').find('LastModified').text
last_modified_gets = [p.find('LastModified').text
for p in elem.iterfind('Part')]
self.assertEquals(
last_modified_get.rsplit('.', 1)[0],
last_modified.rsplit('.', 1)[0])
last_modified_gets[0].rsplit('.', 1)[0],
last_modified_1.rsplit('.', 1)[0],
'%r != %r' % (last_modified_gets[0], last_modified_1))
self.assertEquals(
last_modified_gets[1].rsplit('.', 1)[0],
last_modified_2.rsplit('.', 1)[0],
'%r != %r' % (last_modified_gets[1], last_modified_2))
# There should be *exactly* two parts in the result
self.assertEqual([], last_modified_gets[2:])
# List Parts
key, upload_id = uploads[0]
@ -213,11 +255,12 @@ class TestSwift3MultiUpload(Swift3FunctionalTestCase):
self.assertEquals(elem.find('Bucket').text, bucket)
self.assertEquals(elem.find('Key').text, key)
self.assertEquals(elem.find('UploadId').text, upload_id)
self.assertEquals(u.find('Initiator/ID').text, self.conn.user_id)
self.assertEquals(u.find('Initiator/DisplayName').text,
self.assertEquals(elem.find('Initiator/ID').text, self.conn.user_id)
self.assertEquals(elem.find('Initiator/DisplayName').text,
self.conn.user_id)
self.assertEquals(elem.find('Owner/ID').text, self.conn.user_id)
self.assertEquals(elem.find('Owner/DisplayName').text,
self.conn.user_id)
self.assertEquals(u.find('Owner/ID').text, self.conn.user_id)
self.assertEquals(u.find('Owner/DisplayName').text, self.conn.user_id)
self.assertEquals(elem.find('StorageClass').text, 'STANDARD')
self.assertEquals(elem.find('PartNumberMarker').text, '0')
self.assertEquals(elem.find('NextPartNumberMarker').text, '1')

8
swift3/test/unit/helpers.py
View File

@ -114,9 +114,13 @@ class FakeSwift(object):
if "CONTENT_TYPE" in env:
self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"]
# range requests ought to work, hence conditional_response=True
# range requests ought to work, but copies are special
support_range_and_conditional = not (
method == 'PUT' and
'X-Copy-From' in req.headers and
'Range' in req.headers)
resp = resp_class(req=req, headers=headers, body=body,
conditional_response=True)
conditional_response=support_range_and_conditional)
return resp(env, start_response)
@property

56
swift3/test/unit/test_multi_upload.py
View File

@ -1100,14 +1100,15 @@ class TestSwift3MultiUpload(Swift3TestCase):
self.assertEquals(status.split()[0], '200')
def _test_copy_for_s3acl(self, account, src_permission=None,
src_path='/src_bucket/src_obj',
src_path='/src_bucket/src_obj', src_headers=None,
head_resp=swob.HTTPOk, put_header={}):
owner = 'test:tester'
grants = [Grant(User(account), src_permission)] \
if src_permission else [Grant(User(owner), 'FULL_CONTROL')]
src_o_headers = encode_acl('object', ACL(Owner(owner, owner), grants))
src_o_headers.update({'last-modified': self.last_modified})
self.swift.register('HEAD', '/v1/AUTH_test/src_bucket/src_obj',
src_o_headers.update(src_headers or {})
self.swift.register('HEAD', '/v1/AUTH_test/%s' % src_path.lstrip('/'),
head_resp, src_o_headers, None)
put_headers = {'Authorization': 'AWS %s:hmac' % account,
@ -1323,6 +1324,57 @@ class TestSwift3MultiUpload(Swift3TestCase):
self.assertTrue(headers.get('If-Unmodified-Since') is None)
_, _, headers = self.swift.calls_with_headers[0]
def test_upload_part_copy_range_unsatisfiable(self):
account = 'test:tester'
header = {'X-Amz-Copy-Source-Range': 'bytes=1000-'}
status, header, body = self._test_copy_for_s3acl(
account, src_headers={'Content-Length': '10'}, put_header=header)
self.assertEquals(status.split()[0], '400')
self.assertIn('Range specified is not valid for '
'source object of size: 10', body)
self.assertEquals([
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket+segments/object/X'),
('HEAD', '/v1/AUTH_test/src_bucket/src_obj'),
], self.swift.calls)
def test_upload_part_copy_range_invalid(self):
account = 'test:tester'
header = {'X-Amz-Copy-Source-Range': '0-9'}
status, header, body = \
self._test_copy_for_s3acl(account, put_header=header)
self.assertEquals(status.split()[0], '400', body)
header = {'X-Amz-Copy-Source-Range': 'asdf'}
status, header, body = \
self._test_copy_for_s3acl(account, put_header=header)
self.assertEquals(status.split()[0], '400', body)
def test_upload_part_copy_range(self):
account = 'test:tester'
header = {'X-Amz-Copy-Source-Range': 'bytes=0-9'}
status, header, body = self._test_copy_for_s3acl(
account, src_headers={'Content-Length': '20'}, put_header=header)
self.assertEquals(status.split()[0], '200', body)
self.assertEquals([
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket+segments/object/X'),
('HEAD', '/v1/AUTH_test/src_bucket/src_obj'),
('PUT', '/v1/AUTH_test/bucket+segments/object/X/1'),
], self.swift.calls)
put_headers = self.swift.calls_with_headers[-1][2]
self.assertEquals('bytes=0-9', put_headers['Range'])
self.assertEquals('/src_bucket/src_obj', put_headers['X-Copy-From'])
class TestSwift3MultiUploadNonUTC(TestSwift3MultiUpload):
def setUp(self):

6
swift3/test/unit/test_obj.py
View File

@ -391,6 +391,12 @@ class TestSwift3Obj(Swift3TestCase):
swob.HTTPCreated,
{'X-Amz-Copy-Source': '/bucket/'})
self.assertEquals(code, 'InvalidArgument')
code = self._test_method_error(
'PUT', '/bucket/object',
swob.HTTPCreated,
{'X-Amz-Copy-Source': '/src_bucket/src_object',
'X-Amz-Copy-Source-Range': 'bytes=0-0'})
self.assertEquals(code, 'InvalidArgument')
code = self._test_method_error('PUT', '/bucket/object',
swob.HTTPRequestTimeout)
self.assertEquals(code, 'RequestTimeout')