5320ecbaf2
md5 is not an approved algorithm in FIPS mode, and trying to instantiate a hashlib.md5() will fail when the system is running in FIPS mode. md5 is allowed when in a non-security context. There is a plan to add a keyword parameter (usedforsecurity) to hashlib.md5() to annotate whether or not the instance is being used in a security context. In the case where it is not, the instantiation of md5 will be allowed. See https://bugs.python.org/issue9216 for more details. Some downstream python versions already support this parameter. To support these versions, a new encapsulation of md5() is added to swift/common/utils.py. This encapsulation is identical to the one being added to oslo.utils, but is recreated here to avoid adding a dependency. This patch is to replace the instances of hashlib.md5() with this new encapsulation, adding an annotation indicating whether the usage is a security context or not. While this patch seems large, it is really just the same change over and again. Reviewers need to pay particular attention as to whether the keyword parameter (usedforsecurity) is set correctly. Right now, all of them appear to be not used in a security context. Now that all the instances have been converted, we can update the bandit run to look for these instances and ensure that new invocations do not creep in. With this latest patch, the functional and unit tests all pass on a FIPS enabled system. Co-Authored-By: Pete Zaitcev Change-Id: Ibb4917da4c083e1e094156d748708b87387f2d87
763 lines
30 KiB
Python
763 lines
30 KiB
Python
# Copyright (c) 2019 SwiftStack, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import time
|
|
from collections import defaultdict
|
|
|
|
from botocore.exceptions import ClientError
|
|
import six
|
|
|
|
from swift.common.header_key_dict import HeaderKeyDict
|
|
from swift.common.utils import md5
|
|
from test.s3api import BaseS3TestCase
|
|
|
|
|
|
def retry(f, timeout=10):
|
|
timelimit = time.time() + timeout
|
|
while True:
|
|
try:
|
|
f()
|
|
except (ClientError, AssertionError):
|
|
if time.time() > timelimit:
|
|
raise
|
|
continue
|
|
else:
|
|
break
|
|
|
|
|
|
class TestObjectVersioning(BaseS3TestCase):
|
|
|
|
maxDiff = None
|
|
|
|
def setUp(self):
|
|
self.client = self.get_s3_client(1)
|
|
self.bucket_name = self.create_name('versioning')
|
|
resp = self.client.create_bucket(Bucket=self.bucket_name)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
|
|
def enable_versioning():
|
|
resp = self.client.put_bucket_versioning(
|
|
Bucket=self.bucket_name,
|
|
VersioningConfiguration={'Status': 'Enabled'})
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
retry(enable_versioning)
|
|
|
|
def tearDown(self):
|
|
resp = self.client.put_bucket_versioning(
|
|
Bucket=self.bucket_name,
|
|
VersioningConfiguration={'Status': 'Suspended'})
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.clear_bucket(self.client, self.bucket_name)
|
|
super(TestObjectVersioning, self).tearDown()
|
|
|
|
def test_setup(self):
|
|
bucket_name = self.create_name('new-bucket')
|
|
resp = self.client.create_bucket(Bucket=bucket_name)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
expected_location = '/%s' % bucket_name
|
|
self.assertEqual(expected_location, resp['Location'])
|
|
headers = HeaderKeyDict(resp['ResponseMetadata']['HTTPHeaders'])
|
|
self.assertEqual('0', headers['content-length'])
|
|
self.assertEqual(expected_location, headers['location'])
|
|
|
|
# get versioning
|
|
resp = self.client.get_bucket_versioning(Bucket=bucket_name)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.assertNotIn('Status', resp)
|
|
|
|
# put versioning
|
|
versioning_config = {
|
|
'Status': 'Enabled',
|
|
}
|
|
resp = self.client.put_bucket_versioning(
|
|
Bucket=bucket_name,
|
|
VersioningConfiguration=versioning_config)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
|
|
# ... now it's enabled
|
|
def check_status():
|
|
resp = self.client.get_bucket_versioning(Bucket=bucket_name)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
try:
|
|
self.assertEqual('Enabled', resp['Status'])
|
|
except KeyError:
|
|
self.fail('Status was not in %r' % resp)
|
|
retry(check_status)
|
|
|
|
# send over some bogus junk
|
|
versioning_config['Status'] = 'Disabled'
|
|
with self.assertRaises(ClientError) as ctx:
|
|
self.client.put_bucket_versioning(
|
|
Bucket=bucket_name,
|
|
VersioningConfiguration=versioning_config)
|
|
expected_err = 'An error occurred (MalformedXML) when calling the ' \
|
|
'PutBucketVersioning operation: The XML you provided was ' \
|
|
'not well-formed or did not validate against our published schema'
|
|
self.assertEqual(expected_err, str(ctx.exception))
|
|
|
|
# disable it
|
|
versioning_config['Status'] = 'Suspended'
|
|
resp = self.client.put_bucket_versioning(
|
|
Bucket=bucket_name,
|
|
VersioningConfiguration=versioning_config)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
|
|
# ... now it's disabled again
|
|
def check_status():
|
|
resp = self.client.get_bucket_versioning(Bucket=bucket_name)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.assertEqual('Suspended', resp['Status'])
|
|
retry(check_status)
|
|
|
|
def test_upload_fileobj_versioned(self):
|
|
obj_data = self.create_name('some-data').encode('ascii')
|
|
obj_etag = md5(obj_data, usedforsecurity=False).hexdigest()
|
|
obj_name = self.create_name('versioned-obj')
|
|
self.client.upload_fileobj(six.BytesIO(obj_data),
|
|
self.bucket_name, obj_name)
|
|
|
|
# object is in the listing
|
|
resp = self.client.list_objects_v2(Bucket=self.bucket_name)
|
|
objs = resp.get('Contents', [])
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % obj_etag,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
# object version listing
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
objs = resp.get('Versions', [])
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
obj.pop('Owner')
|
|
obj.pop('VersionId')
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % obj_etag,
|
|
'IsLatest': True,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
# overwrite the object
|
|
new_obj_data = self.create_name('some-new-data').encode('ascii')
|
|
new_obj_etag = md5(new_obj_data, usedforsecurity=False).hexdigest()
|
|
self.client.upload_fileobj(six.BytesIO(new_obj_data),
|
|
self.bucket_name, obj_name)
|
|
|
|
# new object is in the listing
|
|
resp = self.client.list_objects_v2(Bucket=self.bucket_name)
|
|
objs = resp.get('Contents', [])
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % new_obj_etag,
|
|
'Key': obj_name,
|
|
'Size': len(new_obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
# both object versions in the versions listing
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
objs = resp.get('Versions', [])
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
obj.pop('Owner')
|
|
obj.pop('VersionId')
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % new_obj_etag,
|
|
'IsLatest': True,
|
|
'Key': obj_name,
|
|
'Size': len(new_obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}, {
|
|
'ETag': '"%s"' % obj_etag,
|
|
'IsLatest': False,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
def test_delete_versioned_objects(self):
|
|
etags = []
|
|
obj_name = self.create_name('versioned-obj')
|
|
for i in range(3):
|
|
obj_data = self.create_name('some-data-%s' % i).encode('ascii')
|
|
etags.insert(0, md5(obj_data, usedforsecurity=False).hexdigest())
|
|
self.client.upload_fileobj(six.BytesIO(obj_data),
|
|
self.bucket_name, obj_name)
|
|
|
|
# only one object appears in the listing
|
|
resp = self.client.list_objects_v2(Bucket=self.bucket_name)
|
|
objs = resp.get('Contents', [])
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % etags[0],
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
# but everything is layed out in the object versions listing
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
objs = resp.get('Versions', [])
|
|
versions = []
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
obj.pop('Owner')
|
|
versions.append(obj.pop('VersionId'))
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % etags[0],
|
|
'IsLatest': True,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}, {
|
|
'ETag': '"%s"' % etags[1],
|
|
'IsLatest': False,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}, {
|
|
'ETag': '"%s"' % etags[2],
|
|
'IsLatest': False,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
# we can delete a specific version
|
|
resp = self.client.delete_object(Bucket=self.bucket_name,
|
|
Key=obj_name,
|
|
VersionId=versions[1])
|
|
|
|
# and that just pulls it out of the versions listing
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
objs = resp.get('Versions', [])
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
obj.pop('Owner')
|
|
obj.pop('VersionId')
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % etags[0],
|
|
'IsLatest': True,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}, {
|
|
'ETag': '"%s"' % etags[2],
|
|
'IsLatest': False,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
# ... but the current listing is unaffected
|
|
resp = self.client.list_objects_v2(Bucket=self.bucket_name)
|
|
objs = resp.get('Contents', [])
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % etags[0],
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
# OTOH, if you delete specifically the latest version
|
|
# we can delete a specific version
|
|
resp = self.client.delete_object(Bucket=self.bucket_name,
|
|
Key=obj_name,
|
|
VersionId=versions[0])
|
|
|
|
# the versions listing has a new IsLatest
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
objs = resp.get('Versions', [])
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
obj.pop('Owner')
|
|
obj.pop('VersionId')
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % etags[2],
|
|
'IsLatest': True,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
# and the stack pops
|
|
resp = self.client.list_objects_v2(Bucket=self.bucket_name)
|
|
objs = resp.get('Contents', [])
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % etags[2],
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
def test_delete_versioned_deletes(self):
|
|
etags = []
|
|
obj_name = self.create_name('versioned-obj')
|
|
for i in range(3):
|
|
obj_data = self.create_name('some-data-%s' % i).encode('ascii')
|
|
etags.insert(0, md5(obj_data, usedforsecurity=False).hexdigest())
|
|
self.client.upload_fileobj(six.BytesIO(obj_data),
|
|
self.bucket_name, obj_name)
|
|
# and make a delete marker
|
|
self.client.delete_object(Bucket=self.bucket_name, Key=obj_name)
|
|
|
|
# current listing is empty
|
|
resp = self.client.list_objects_v2(Bucket=self.bucket_name)
|
|
objs = resp.get('Contents', [])
|
|
self.assertEqual([], objs)
|
|
|
|
# but everything is in layed out in the versions listing
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
objs = resp.get('Versions', [])
|
|
versions = []
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
obj.pop('Owner')
|
|
versions.append(obj.pop('VersionId'))
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % etag,
|
|
'IsLatest': False,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
} for etag in etags], objs)
|
|
# ... plus the delete markers
|
|
delete_markers = resp.get('DeleteMarkers', [])
|
|
marker_versions = []
|
|
for marker in delete_markers:
|
|
marker.pop('LastModified')
|
|
marker.pop('Owner')
|
|
marker_versions.append(marker.pop('VersionId'))
|
|
self.assertEqual([{
|
|
'Key': obj_name,
|
|
'IsLatest': is_latest,
|
|
} for is_latest in (True, False, False)], delete_markers)
|
|
|
|
# delete an old delete markers
|
|
resp = self.client.delete_object(Bucket=self.bucket_name,
|
|
Key=obj_name,
|
|
VersionId=marker_versions[2])
|
|
|
|
# since IsLatest is still marker we'll raise NoSuchKey
|
|
with self.assertRaises(ClientError) as caught:
|
|
resp = self.client.get_object(Bucket=self.bucket_name,
|
|
Key=obj_name)
|
|
expected_err = 'An error occurred (NoSuchKey) when calling the ' \
|
|
'GetObject operation: The specified key does not exist.'
|
|
self.assertEqual(expected_err, str(caught.exception))
|
|
|
|
# now delete the delete marker (IsLatest)
|
|
resp = self.client.delete_object(Bucket=self.bucket_name,
|
|
Key=obj_name,
|
|
VersionId=marker_versions[0])
|
|
|
|
# most recent version is now latest
|
|
resp = self.client.get_object(Bucket=self.bucket_name,
|
|
Key=obj_name)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.assertEqual('"%s"' % etags[0], resp['ETag'])
|
|
|
|
# now delete the IsLatest object version
|
|
resp = self.client.delete_object(Bucket=self.bucket_name,
|
|
Key=obj_name,
|
|
VersionId=versions[0])
|
|
|
|
# and object is deleted again
|
|
with self.assertRaises(ClientError) as caught:
|
|
resp = self.client.get_object(Bucket=self.bucket_name,
|
|
Key=obj_name)
|
|
expected_err = 'An error occurred (NoSuchKey) when calling the ' \
|
|
'GetObject operation: The specified key does not exist.'
|
|
self.assertEqual(expected_err, str(caught.exception))
|
|
|
|
# delete marker IsLatest
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
delete_markers = resp.get('DeleteMarkers', [])
|
|
for marker in delete_markers:
|
|
marker.pop('LastModified')
|
|
marker.pop('Owner')
|
|
self.assertEqual([{
|
|
'Key': obj_name,
|
|
'IsLatest': True,
|
|
'VersionId': marker_versions[1],
|
|
}], delete_markers)
|
|
|
|
def test_multipart_upload(self):
|
|
obj_name = self.create_name('versioned-obj')
|
|
obj_data = b'data'
|
|
|
|
mu = self.client.create_multipart_upload(
|
|
Bucket=self.bucket_name,
|
|
Key=obj_name)
|
|
part_md5 = self.client.upload_part(
|
|
Bucket=self.bucket_name,
|
|
Key=obj_name,
|
|
UploadId=mu['UploadId'],
|
|
PartNumber=1,
|
|
Body=obj_data)['ETag']
|
|
complete_response = self.client.complete_multipart_upload(
|
|
Bucket=self.bucket_name,
|
|
Key=obj_name,
|
|
UploadId=mu['UploadId'],
|
|
MultipartUpload={'Parts': [
|
|
{'PartNumber': 1, 'ETag': part_md5},
|
|
]})
|
|
obj_etag = complete_response['ETag']
|
|
|
|
delete_response = self.client.delete_object(
|
|
Bucket=self.bucket_name,
|
|
Key=obj_name)
|
|
marker_version_id = delete_response['VersionId']
|
|
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
objs = resp.get('Versions', [])
|
|
versions = []
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
obj.pop('Owner')
|
|
versions.append(obj.pop('VersionId'))
|
|
self.assertEqual([{
|
|
'ETag': obj_etag,
|
|
'IsLatest': False,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
markers = resp.get('DeleteMarkers', [])
|
|
for marker in markers:
|
|
marker.pop('LastModified')
|
|
marker.pop('Owner')
|
|
self.assertEqual([{
|
|
'IsLatest': True,
|
|
'Key': obj_name,
|
|
'VersionId': marker_version_id,
|
|
}], markers)
|
|
|
|
# Can still get the old version
|
|
resp = self.client.get_object(
|
|
Bucket=self.bucket_name,
|
|
Key=obj_name,
|
|
VersionId=versions[0])
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.assertEqual(obj_etag, resp['ETag'])
|
|
|
|
delete_response = self.client.delete_object(
|
|
Bucket=self.bucket_name,
|
|
Key=obj_name,
|
|
VersionId=versions[0])
|
|
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
self.assertEqual([], resp.get('Versions', []))
|
|
|
|
markers = resp.get('DeleteMarkers', [])
|
|
for marker in markers:
|
|
marker.pop('LastModified')
|
|
marker.pop('Owner')
|
|
self.assertEqual([{
|
|
'IsLatest': True,
|
|
'Key': obj_name,
|
|
'VersionId': marker_version_id,
|
|
}], markers)
|
|
|
|
def test_get_versioned_object(self):
|
|
etags = []
|
|
obj_name = self.create_name('versioned-obj')
|
|
for i in range(3):
|
|
obj_data = self.create_name('some-data-%s' % i).encode('ascii')
|
|
# TODO: pull etag from response instead
|
|
etags.insert(0, md5(obj_data, usedforsecurity=False).hexdigest())
|
|
self.client.upload_fileobj(
|
|
six.BytesIO(obj_data), self.bucket_name, obj_name)
|
|
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
objs = resp.get('Versions', [])
|
|
versions = []
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
obj.pop('Owner')
|
|
versions.append(obj.pop('VersionId'))
|
|
self.assertEqual([{
|
|
'ETag': '"%s"' % etags[0],
|
|
'IsLatest': True,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}, {
|
|
'ETag': '"%s"' % etags[1],
|
|
'IsLatest': False,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}, {
|
|
'ETag': '"%s"' % etags[2],
|
|
'IsLatest': False,
|
|
'Key': obj_name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
}], objs)
|
|
|
|
# un-versioned get_object returns IsLatest
|
|
resp = self.client.get_object(Bucket=self.bucket_name, Key=obj_name)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.assertEqual('"%s"' % etags[0], resp['ETag'])
|
|
|
|
# but you can get any object by version
|
|
for i, version in enumerate(versions):
|
|
resp = self.client.get_object(
|
|
Bucket=self.bucket_name, Key=obj_name, VersionId=version)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.assertEqual('"%s"' % etags[i], resp['ETag'])
|
|
|
|
# and head_object works about the same
|
|
resp = self.client.head_object(Bucket=self.bucket_name, Key=obj_name)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.assertEqual('"%s"' % etags[0], resp['ETag'])
|
|
self.assertEqual(versions[0], resp['VersionId'])
|
|
for version, etag in zip(versions, etags):
|
|
resp = self.client.head_object(
|
|
Bucket=self.bucket_name, Key=obj_name, VersionId=version)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.assertEqual(version, resp['VersionId'])
|
|
self.assertEqual('"%s"' % etag, resp['ETag'])
|
|
|
|
def test_get_versioned_object_invalid_params(self):
|
|
with self.assertRaises(ClientError) as ctx:
|
|
self.client.list_object_versions(Bucket=self.bucket_name,
|
|
KeyMarker='',
|
|
VersionIdMarker='bogus')
|
|
expected_err = 'An error occurred (InvalidArgument) when calling ' \
|
|
'the ListObjectVersions operation: Invalid version id specified'
|
|
self.assertEqual(expected_err, str(ctx.exception))
|
|
|
|
with self.assertRaises(ClientError) as ctx:
|
|
self.client.list_object_versions(
|
|
Bucket=self.bucket_name,
|
|
VersionIdMarker='a' * 32)
|
|
expected_err = 'An error occurred (InvalidArgument) when calling ' \
|
|
'the ListObjectVersions operation: A version-id marker cannot ' \
|
|
'be specified without a key marker.'
|
|
self.assertEqual(expected_err, str(ctx.exception))
|
|
|
|
def test_get_versioned_object_key_marker(self):
|
|
obj00_name = self.create_name('00-versioned-obj')
|
|
obj01_name = self.create_name('01-versioned-obj')
|
|
names = [obj00_name] * 3 + [obj01_name] * 3
|
|
latest = [True, False, False, True, False, False]
|
|
etags = []
|
|
for i in range(3):
|
|
obj_data = self.create_name('some-data-%s' % i).encode('ascii')
|
|
etags.insert(0, '"%s"' % md5(
|
|
obj_data, usedforsecurity=False).hexdigest())
|
|
self.client.upload_fileobj(
|
|
six.BytesIO(obj_data), self.bucket_name, obj01_name)
|
|
for i in range(3):
|
|
obj_data = self.create_name('some-data-%s' % i).encode('ascii')
|
|
etags.insert(0, '"%s"' % md5(
|
|
obj_data, usedforsecurity=False).hexdigest())
|
|
self.client.upload_fileobj(
|
|
six.BytesIO(obj_data), self.bucket_name, obj00_name)
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
versions = []
|
|
objs = []
|
|
for o in resp.get('Versions', []):
|
|
versions.append(o['VersionId'])
|
|
objs.append({
|
|
'Key': o['Key'],
|
|
'VersionId': o['VersionId'],
|
|
'IsLatest': o['IsLatest'],
|
|
'ETag': o['ETag'],
|
|
})
|
|
expected = [{
|
|
'Key': name,
|
|
'VersionId': version,
|
|
'IsLatest': is_latest,
|
|
'ETag': etag,
|
|
} for name, etag, version, is_latest in zip(
|
|
names, etags, versions, latest)]
|
|
self.assertEqual(expected, objs)
|
|
|
|
# on s3 this makes expected[0]['IsLatest'] magicaly change to False?
|
|
# resp = self.client.list_object_versions(Bucket=self.bucket_name,
|
|
# KeyMarker='',
|
|
# VersionIdMarker=versions[0])
|
|
# objs = [{
|
|
# 'Key': o['Key'],
|
|
# 'VersionId': o['VersionId'],
|
|
# 'IsLatest': o['IsLatest'],
|
|
# 'ETag': o['ETag'],
|
|
# } for o in resp.get('Versions', [])]
|
|
# self.assertEqual(expected, objs)
|
|
|
|
# KeyMarker skips past that key
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name,
|
|
KeyMarker=obj00_name)
|
|
objs = [{
|
|
'Key': o['Key'],
|
|
'VersionId': o['VersionId'],
|
|
'IsLatest': o['IsLatest'],
|
|
'ETag': o['ETag'],
|
|
} for o in resp.get('Versions', [])]
|
|
self.assertEqual(expected[3:], objs)
|
|
|
|
# KeyMarker with VersionIdMarker skips past that version
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name,
|
|
KeyMarker=obj00_name,
|
|
VersionIdMarker=versions[0])
|
|
objs = [{
|
|
'Key': o['Key'],
|
|
'VersionId': o['VersionId'],
|
|
'IsLatest': o['IsLatest'],
|
|
'ETag': o['ETag'],
|
|
} for o in resp.get('Versions', [])]
|
|
self.assertEqual(expected[1:], objs)
|
|
|
|
# KeyMarker with bogus version skips past that key
|
|
resp = self.client.list_object_versions(
|
|
Bucket=self.bucket_name,
|
|
KeyMarker=obj00_name,
|
|
VersionIdMarker=versions[4])
|
|
objs = [{
|
|
'Key': o['Key'],
|
|
'VersionId': o['VersionId'],
|
|
'IsLatest': o['IsLatest'],
|
|
'ETag': o['ETag'],
|
|
} for o in resp.get('Versions', [])]
|
|
self.assertEqual(expected[3:], objs)
|
|
|
|
def test_list_objects(self):
|
|
etags = defaultdict(list)
|
|
for i in range(3):
|
|
obj_name = self.create_name('versioned-obj')
|
|
for i in range(3):
|
|
obj_data = self.create_name('some-data-%s' % i).encode('ascii')
|
|
etags[obj_name].insert(0, md5(
|
|
obj_data, usedforsecurity=False).hexdigest())
|
|
self.client.upload_fileobj(
|
|
six.BytesIO(obj_data), self.bucket_name, obj_name)
|
|
|
|
# both unversioned list_objects responses are similar
|
|
expected = []
|
|
for name, obj_etags in sorted(etags.items()):
|
|
expected.append({
|
|
'ETag': '"%s"' % obj_etags[0],
|
|
'Key': name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
})
|
|
resp = self.client.list_objects(Bucket=self.bucket_name)
|
|
objs = resp.get('Contents', [])
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
# one difference seems to be the Owner key
|
|
self.assertEqual({'DisplayName', 'ID'},
|
|
set(obj.pop('Owner').keys()))
|
|
self.assertEqual(expected, objs)
|
|
resp = self.client.list_objects_v2(Bucket=self.bucket_name)
|
|
objs = resp.get('Contents', [])
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
self.assertEqual(expected, objs)
|
|
|
|
# versioned listings has something for everyone
|
|
expected = []
|
|
for name, obj_etags in sorted(etags.items()):
|
|
is_latest = True
|
|
for etag in obj_etags:
|
|
expected.append({
|
|
'ETag': '"%s"' % etag,
|
|
'IsLatest': is_latest,
|
|
'Key': name,
|
|
'Size': len(obj_data),
|
|
'StorageClass': 'STANDARD',
|
|
})
|
|
is_latest = False
|
|
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
objs = resp.get('Versions', [])
|
|
versions = []
|
|
for obj in objs:
|
|
obj.pop('LastModified')
|
|
obj.pop('Owner')
|
|
versions.append(obj.pop('VersionId'))
|
|
self.assertEqual(expected, objs)
|
|
|
|
def test_copy_object(self):
|
|
etags = []
|
|
obj_name = self.create_name('versioned-obj')
|
|
for i in range(3):
|
|
obj_data = self.create_name('some-data-%s' % i).encode('ascii')
|
|
etags.insert(0, md5(
|
|
obj_data, usedforsecurity=False).hexdigest())
|
|
self.client.upload_fileobj(
|
|
six.BytesIO(obj_data), self.bucket_name, obj_name)
|
|
|
|
resp = self.client.list_object_versions(Bucket=self.bucket_name)
|
|
objs = resp.get('Versions', [])
|
|
versions = []
|
|
for obj in objs:
|
|
versions.append(obj.pop('VersionId'))
|
|
|
|
# CopySource can just be Bucket/Key string
|
|
first_target = self.create_name('target-obj1')
|
|
copy_resp = self.client.copy_object(
|
|
Bucket=self.bucket_name, Key=first_target,
|
|
CopySource='%s/%s' % (self.bucket_name, obj_name))
|
|
self.assertEqual(versions[0], copy_resp['CopySourceVersionId'])
|
|
|
|
# and you'll just get the most recent version
|
|
resp = self.client.head_object(Bucket=self.bucket_name,
|
|
Key=first_target)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.assertEqual('"%s"' % etags[0], resp['ETag'])
|
|
|
|
# or you can be more explicit
|
|
explicit_target = self.create_name('target-%s' % versions[0])
|
|
copy_source = {'Bucket': self.bucket_name, 'Key': obj_name,
|
|
'VersionId': versions[0]}
|
|
copy_resp = self.client.copy_object(
|
|
Bucket=self.bucket_name, Key=explicit_target,
|
|
CopySource=copy_source)
|
|
self.assertEqual(versions[0], copy_resp['CopySourceVersionId'])
|
|
# and you still get the same thing
|
|
resp = self.client.head_object(Bucket=self.bucket_name,
|
|
Key=explicit_target)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.assertEqual('"%s"' % etags[0], resp['ETag'])
|
|
|
|
# but you can also copy from a specific version
|
|
version_target = self.create_name('target-%s' % versions[2])
|
|
copy_source['VersionId'] = versions[2]
|
|
copy_resp = self.client.copy_object(
|
|
Bucket=self.bucket_name, Key=version_target,
|
|
CopySource=copy_source)
|
|
self.assertEqual(versions[2], copy_resp['CopySourceVersionId'])
|
|
resp = self.client.head_object(Bucket=self.bucket_name,
|
|
Key=version_target)
|
|
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
|
self.assertEqual('"%s"' % etags[2], resp['ETag'])
|