From d4e7940457cd31b4ee91a01a0c492781179fcfe4 Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Sat, 4 May 2019 12:11:54 -0700 Subject: [PATCH] Start transition to boto3 in s3api tests. Adds the scaffolding required for tests to use boto3 and converts the test_bucket.py tests to the new interface. Follow on patches will convert the other tests to use the boto3 library. Notable changes: we no longer try to reach for the equivalent of `boto.make_request()` and instead rely on the boto3/botocore event system to mutate requests as necessary (or to disable pre-flight validators). Partial-Bug: 1557260 Change-Id: I3d77ef4a6b878c49ebfa0c8b8647d7199d87601e --- doc/requirements.txt | 3 +- lower-constraints.txt | 2 + test-requirements.txt | 2 + test/functional/s3api/__init__.py | 23 +- test/functional/s3api/s3_test_client.py | 49 +++ test/functional/s3api/test_bucket.py | 553 +++++++++++------------- 6 files changed, 322 insertions(+), 310 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 13c167a85b..c5f6173b16 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -2,7 +2,8 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. # this is required for the docs build jobs -sphinx>=1.6.2 # BSD +sphinx>=1.6.2,<2.0.0;python_version=='2.7' # BSD +sphinx>=1.6.2;python_version>='3.4' # BSD openstackdocstheme>=1.11.0 # Apache-2.0 reno>=1.8.0 # Apache-2.0 os-api-ref>=1.0.0 # Apache-2.0 diff --git a/lower-constraints.txt b/lower-constraints.txt index 2d89f16625..e30c1b53f1 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -3,6 +3,8 @@ asn1crypto==0.24.0 Babel==2.5.3 bandit==1.1.0 boto==2.32.1 +boto3==1.9 +botocore==1.12 castellan==0.13.0 certifi==2018.1.18 cffi==1.11.5 diff --git a/test-requirements.txt b/test-requirements.txt index cf97918f03..ff86a01dd1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,6 +15,8 @@ python-keystoneclient!=2.1.0,>=2.0.0 # Apache-2.0 reno>=1.8.0 # Apache-2.0 python-openstackclient>=3.12.0 boto>=2.32.1 +boto3>=1.9 +botocore>=1.12 requests-mock>=1.2.0 # Apache-2.0 fixtures>=3.0.0 # Apache-2.0/BSD keystonemiddleware>=4.17.0 # Apache-2.0 diff --git a/test/functional/s3api/__init__.py b/test/functional/s3api/__init__.py index 9fcd0a201c..4cec6dd9b0 100644 --- a/test/functional/s3api/__init__.py +++ b/test/functional/s3api/__init__.py @@ -16,7 +16,8 @@ import unittest2 import traceback import test.functional as tf -from test.functional.s3api.s3_test_client import Connection +from test.functional.s3api.s3_test_client import ( + Connection, get_boto3_conn, tear_down_s3) def setUpModule(): @@ -59,3 +60,23 @@ class S3ApiBase(unittest2.TestCase): if etag is not None: self.assertTrue('etag' in headers) # sanity self.assertEqual(etag, headers['etag'].strip('"')) + + +class S3ApiBaseBoto3(S3ApiBase): + def setUp(self): + if 's3api' not in tf.cluster_info: + raise tf.SkipTest('s3api middleware is not enabled') + try: + self.conn = get_boto3_conn() + self.endpoint_url = self.conn._endpoint.host + self.access_key = self.conn._request_signer._credentials.access_key + self.region = self.conn._client_config.region_name + tear_down_s3(self.conn) + except Exception: + message = '%s got an error during initialize process.\n\n%s' % \ + (self.method_name, traceback.format_exc()) + # TODO: Find a way to make this go to FAIL instead of Error + self.fail(message) + + def tearDown(self): + tear_down_s3(self.conn) diff --git a/test/functional/s3api/s3_test_client.py b/test/functional/s3api/s3_test_client.py index 8ceccbff4b..4736e1d240 100644 --- a/test/functional/s3api/s3_test_client.py +++ b/test/functional/s3api/s3_test_client.py @@ -15,6 +15,8 @@ import os import test.functional as tf +import boto3 +from botocore.exceptions import ClientError from boto.s3.connection import S3Connection, OrdinaryCallingFormat, \ S3ResponseError import six @@ -135,6 +137,53 @@ class Connection(object): return url, {} +def get_boto3_conn(aws_access_key='test:tester', aws_secret_key='testing'): + host = tf.config['auth_host'] + port = int(tf.config['auth_port']) + config = boto3.session.Config(s3={'addressing_style': 'path'}) + return boto3.client( + 's3', aws_access_key_id=aws_access_key, + aws_secret_access_key=aws_secret_key, + config=config, region_name='us-east-1', use_ssl=False, + endpoint_url='http://{}:{}'.format(host, port)) + + +def tear_down_s3(conn): + """ + Reset all swift environment to keep clean. As a result by calling this + method, we can assume the backend swift keeps no containers and no + objects on this connection's account. + """ + exceptions = [] + for i in range(RETRY_COUNT): + try: + resp = conn.list_buckets() + buckets = [bucket['Name'] for bucket in resp.get('Buckets', [])] + for bucket in buckets: + try: + resp = conn.list_multipart_uploads(Bucket=bucket) + for upload in resp.get('Uploads', []): + conn.abort_multipart_upload( + Bucket=bucket, + Key=upload['Key'], + UploadId=upload['UploadId']) + + resp = conn.list_objects(Bucket=bucket) + for obj in resp.get('Contents', []): + conn.delete_object(Bucket=bucket, Key=obj['Key']) + conn.delete_bucket(Bucket=bucket) + except ClientError as e: + # 404 means NoSuchBucket, NoSuchKey, or NoSuchUpload + if e.response['ResponseMetadata']['HTTPStatusCode'] != 404: + raise + except Exception as e: + exceptions.append(''.join( + traceback.format_exception(*sys.exc_info()))) + if exceptions: + exceptions.insert(0, 'Too many errors to continue:') + raise Exception('\n========\n'.join(exceptions)) + + # TODO: make sure where this function is used def get_admin_connection(): """ diff --git a/test/functional/s3api/test_bucket.py b/test/functional/s3api/test_bucket.py index cec0053b9f..c1f64e6ec6 100644 --- a/test/functional/s3api/test_bucket.py +++ b/test/functional/s3api/test_bucket.py @@ -13,16 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import botocore +import datetime import unittest2 import os import test.functional as tf from swift.common.utils import config_true_value -from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \ - SubElement -from test.functional.s3api import S3ApiBase -from test.functional.s3api.s3_test_client import Connection -from test.functional.s3api.utils import get_error_code +from test.functional.s3api import S3ApiBaseBoto3 +from test.functional.s3api.s3_test_client import get_boto3_conn def setUpModule(): @@ -33,14 +32,21 @@ def tearDownModule(): tf.teardown_package() -class TestS3ApiBucket(S3ApiBase): - def setUp(self): - super(TestS3ApiBucket, self).setUp() - - def _gen_location_xml(self, location): - elem = Element('CreateBucketConfiguration') - SubElement(elem, 'LocationConstraint').text = location - return tostring(elem) +class TestS3ApiBucket(S3ApiBaseBoto3): + def _validate_object_listing(self, resp_objects, req_objects, + expect_owner=True): + self.assertEqual(len(resp_objects), len(req_objects)) + for i, obj in enumerate(resp_objects): + self.assertEqual(obj['Key'], req_objects[i]) + self.assertEqual(type(obj['LastModified']), datetime.datetime) + self.assertIn('ETag', obj) + self.assertIn('Size', obj) + self.assertEqual(obj['StorageClass'], 'STANDARD') + if expect_owner: + self.assertEqual(obj['Owner']['ID'], self.access_key) + self.assertEqual(obj['Owner']['DisplayName'], self.access_key) + else: + self.assertNotIn('Owner', obj) def test_bucket(self): bucket = 'bucket' @@ -48,112 +54,95 @@ class TestS3ApiBucket(S3ApiBase): 'max_bucket_listing', 1000) # PUT Bucket - status, headers, body = self.conn.make_request('PUT', bucket) - self.assertEqual(status, 200) + resp = self.conn.create_bucket(Bucket=bucket) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + headers = resp['ResponseMetadata']['HTTPHeaders'] self.assertCommonResponseHeaders(headers) self.assertIn(headers['location'], ( '/' + bucket, # swob won't touch it... # but webob (which we get because of auth_token) *does* - 'http://%s%s/%s' % ( - self.conn.host, - '' if self.conn.port == 80 else ':%d' % self.conn.port, - bucket), - # This is all based on the Host header the client provided, - # and boto will double-up ports for sig v4. See - # - https://github.com/boto/boto/issues/2623 - # - https://github.com/boto/boto/issues/3716 - # with proposed fixes at - # - https://github.com/boto/boto/pull/3513 - # - https://github.com/boto/boto/pull/3676 - 'http://%s%s:%d/%s' % ( - self.conn.host, - '' if self.conn.port == 80 else ':%d' % self.conn.port, - self.conn.port, - bucket), + '%s/%s' % (self.endpoint_url, bucket), )) self.assertEqual(headers['content-length'], '0') # GET Bucket(Without Object) - status, headers, body = self.conn.make_request('GET', bucket) - self.assertEqual(status, 200) + resp = self.conn.list_objects(Bucket=bucket) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + headers = resp['ResponseMetadata']['HTTPHeaders'] self.assertCommonResponseHeaders(headers) self.assertIsNotNone(headers['content-type']) - self.assertEqual(headers['content-length'], str(len(body))) # TODO; requires consideration # self.assertEqual(headers['transfer-encoding'], 'chunked') - elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('Name').text, bucket) - self.assertIsNone(elem.find('Prefix').text) - self.assertIsNone(elem.find('Marker').text) - self.assertEqual( - elem.find('MaxKeys').text, str(max_bucket_listing)) - self.assertEqual(elem.find('IsTruncated').text, 'false') - objects = elem.findall('./Contents') - self.assertEqual(list(objects), []) + self.assertEqual(resp['Name'], bucket) + self.assertEqual(resp['Prefix'], '') + self.assertEqual(resp['Marker'], '') + self.assertEqual(resp['MaxKeys'], max_bucket_listing) + self.assertFalse(resp['IsTruncated']) + self.assertNotIn('Contents', bucket) # GET Bucket(With Object) - req_objects = ('object', 'object2') + req_objects = ['object', 'object2'] for obj in req_objects: - self.conn.make_request('PUT', bucket, obj) - status, headers, body = self.conn.make_request('GET', bucket) - self.assertEqual(status, 200) + self.conn.put_object(Bucket=bucket, Key=obj, Body=b'') + resp = self.conn.list_objects(Bucket=bucket) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) - elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('Name').text, bucket) - self.assertIsNone(elem.find('Prefix').text) - self.assertIsNone(elem.find('Marker').text) - self.assertEqual(elem.find('MaxKeys').text, - str(max_bucket_listing)) - self.assertEqual(elem.find('IsTruncated').text, 'false') - resp_objects = elem.findall('./Contents') - self.assertEqual(len(list(resp_objects)), 2) - for o in resp_objects: - self.assertIn(o.find('Key').text, req_objects) - self.assertIsNotNone(o.find('LastModified').text) - self.assertRegexpMatches( - o.find('LastModified').text, - r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') - self.assertIsNotNone(o.find('ETag').text) - self.assertIsNotNone(o.find('Size').text) - self.assertIsNotNone(o.find('StorageClass').text) - self.assertEqual(o.find('Owner/ID').text, self.conn.user_id) - self.assertEqual(o.find('Owner/DisplayName').text, - self.conn.user_id) + self.assertEqual(resp['Name'], bucket) + self.assertEqual(resp['Prefix'], '') + self.assertEqual(resp['Marker'], '') + self.assertEqual(resp['MaxKeys'], max_bucket_listing) + self.assertFalse(resp['IsTruncated']) + self._validate_object_listing(resp['Contents'], req_objects) # HEAD Bucket - status, headers, body = self.conn.make_request('HEAD', bucket) - self.assertEqual(status, 200) + resp = self.conn.head_bucket(Bucket=bucket) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + headers = resp['ResponseMetadata']['HTTPHeaders'] self.assertCommonResponseHeaders(headers) self.assertIsNotNone(headers['content-type']) - self.assertEqual(headers['content-length'], str(len(body))) # TODO; requires consideration # self.assertEqual(headers['transfer-encoding'], 'chunked') # DELETE Bucket for obj in req_objects: - self.conn.make_request('DELETE', bucket, obj) - status, headers, body = self.conn.make_request('DELETE', bucket) - self.assertEqual(status, 204) + self.conn.delete_object(Bucket=bucket, Key=obj) + resp = self.conn.delete_bucket(Bucket=bucket) + self.assertEqual(204, resp['ResponseMetadata']['HTTPStatusCode']) - self.assertCommonResponseHeaders(headers) + self.assertCommonResponseHeaders( + resp['ResponseMetadata']['HTTPHeaders']) def test_put_bucket_error(self): - status, headers, body = \ - self.conn.make_request('PUT', 'bucket+invalid') - self.assertEqual(get_error_code(body), 'InvalidBucketName') + event_system = self.conn.meta.events + event_system.unregister( + 'before-parameter-build.s3', + botocore.handlers.validate_bucket_name) + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + self.conn.create_bucket(Bucket='bucket+invalid') + self.assertEqual( + ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 400) + self.assertEqual( + ctx.exception.response['Error']['Code'], 'InvalidBucketName') - auth_error_conn = Connection(aws_secret_key='invalid') - status, headers, body = auth_error_conn.make_request('PUT', 'bucket') - self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch') + auth_error_conn = get_boto3_conn(aws_secret_key='invalid') + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + auth_error_conn.create_bucket(Bucket='bucket') + self.assertEqual( + ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 403) + self.assertEqual(ctx.exception.response['Error']['Code'], + 'SignatureDoesNotMatch') - self.conn.make_request('PUT', 'bucket') - status, headers, body = self.conn.make_request('PUT', 'bucket') - self.assertEqual(status, 409) - self.assertEqual(get_error_code(body), 'BucketAlreadyOwnedByYou') + self.conn.create_bucket(Bucket='bucket') + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + self.conn.create_bucket(Bucket='bucket') + self.assertEqual( + ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 409) + self.assertEqual( + ctx.exception.response['Error']['Code'], 'BucketAlreadyOwnedByYou') def test_put_bucket_error_key2(self): if config_true_value(tf.cluster_info['s3api'].get('s3_acl')): @@ -163,15 +152,18 @@ class TestS3ApiBucket(S3ApiBase): 'Cannot test for BucketAlreadyExists with second user; ' 'need s3_access_key2 and s3_secret_key2 configured') - self.conn.make_request('PUT', 'bucket') + self.conn.create_bucket(Bucket='bucket') # Other users of the same account get the same 409 error - conn2 = Connection(tf.config['s3_access_key2'], - tf.config['s3_secret_key2'], - tf.config['s3_access_key2']) - status, headers, body = conn2.make_request('PUT', 'bucket') - self.assertEqual(status, 409) - self.assertEqual(get_error_code(body), 'BucketAlreadyExists') + conn2 = get_boto3_conn(tf.config['s3_access_key2'], + tf.config['s3_secret_key2']) + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + conn2.create_bucket(Bucket='bucket') + self.assertEqual( + ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], + 409) + self.assertEqual( + ctx.exception.response['Error']['Code'], 'BucketAlreadyExists') def test_put_bucket_error_key3(self): if 's3_access_key3' not in tf.config or \ @@ -179,41 +171,51 @@ class TestS3ApiBucket(S3ApiBase): raise tf.SkipTest('Cannot test for AccessDenied; need ' 's3_access_key3 and s3_secret_key3 configured') - self.conn.make_request('PUT', 'bucket') + self.conn.create_bucket(Bucket='bucket') # If the user can't create buckets, they shouldn't even know # whether the bucket exists. - conn3 = Connection(tf.config['s3_access_key3'], - tf.config['s3_secret_key3'], - tf.config['s3_access_key3']) - status, headers, body = conn3.make_request('PUT', 'bucket') - self.assertEqual(status, 403) - self.assertEqual(get_error_code(body), 'AccessDenied') + conn3 = get_boto3_conn(tf.config['s3_access_key3'], + tf.config['s3_secret_key3']) + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + conn3.create_bucket(Bucket='bucket') + self.assertEqual( + ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 403) + self.assertEqual( + ctx.exception.response['Error']['Code'], 'AccessDenied') def test_put_bucket_with_LocationConstraint(self): - bucket = 'bucket' - xml = self._gen_location_xml(self.conn.conn.auth_region_name) - status, headers, body = \ - self.conn.make_request('PUT', bucket, body=xml) - self.assertEqual(status, 200) + resp = self.conn.create_bucket( + Bucket='bucket', + CreateBucketConfiguration={'LocationConstraint': self.region}) + self.assertEqual(resp['ResponseMetadata']['HTTPStatusCode'], 200) def test_get_bucket_error(self): - self.conn.make_request('PUT', 'bucket') + event_system = self.conn.meta.events + event_system.unregister( + 'before-parameter-build.s3', + botocore.handlers.validate_bucket_name) + self.conn.create_bucket(Bucket='bucket') - status, headers, body = \ - self.conn.make_request('GET', 'bucket+invalid') - self.assertEqual(get_error_code(body), 'InvalidBucketName') + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + self.conn.list_objects(Bucket='bucket+invalid') + self.assertEqual( + ctx.exception.response['Error']['Code'], 'InvalidBucketName') - auth_error_conn = Connection(aws_secret_key='invalid') - status, headers, body = auth_error_conn.make_request('GET', 'bucket') - self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch') + auth_error_conn = get_boto3_conn(aws_secret_key='invalid') + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + auth_error_conn.list_objects(Bucket='bucket') + self.assertEqual( + ctx.exception.response['Error']['Code'], 'SignatureDoesNotMatch') - status, headers, body = self.conn.make_request('GET', 'nothing') - self.assertEqual(get_error_code(body), 'NoSuchBucket') + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + self.conn.list_objects(Bucket='nothing') + self.assertEqual( + ctx.exception.response['Error']['Code'], 'NoSuchBucket') def _prepare_test_get_bucket(self, bucket, objects): - self.conn.make_request('PUT', bucket) + self.conn.create_bucket(Bucket=bucket) for obj in objects: - self.conn.make_request('PUT', bucket, obj) + self.conn.put_object(Bucket=bucket, Key=obj, Body=b'') def test_get_bucket_with_delimiter(self): bucket = 'bucket' @@ -222,32 +224,16 @@ class TestS3ApiBucket(S3ApiBase): self._prepare_test_get_bucket(bucket, put_objects) delimiter = '/' - query = 'delimiter=%s' % delimiter expect_objects = ('object', 'object2') expect_prefixes = ('dir/', 'subdir/', 'subdir2/') - status, headers, body = \ - self.conn.make_request('GET', bucket, query=query) - self.assertEqual(status, 200) - elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('Delimiter').text, delimiter) - resp_objects = elem.findall('./Contents') - self.assertEqual(len(list(resp_objects)), len(expect_objects)) - for i, o in enumerate(resp_objects): - self.assertEqual(o.find('Key').text, expect_objects[i]) - self.assertIsNotNone(o.find('LastModified').text) - self.assertRegexpMatches( - o.find('LastModified').text, - r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') - self.assertIsNotNone(o.find('ETag').text) - self.assertIsNotNone(o.find('Size').text) - self.assertEqual(o.find('StorageClass').text, 'STANDARD') - self.assertEqual(o.find('Owner/ID').text, self.conn.user_id) - self.assertEqual(o.find('Owner/DisplayName').text, - self.conn.user_id) - resp_prefixes = elem.findall('CommonPrefixes') - self.assertEqual(len(resp_prefixes), len(expect_prefixes)) - for i, p in enumerate(resp_prefixes): - self.assertEqual(p.find('./Prefix').text, expect_prefixes[i]) + resp = self.conn.list_objects(Bucket=bucket, Delimiter=delimiter) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(resp['Delimiter'], delimiter) + self._validate_object_listing(resp['Contents'], expect_objects) + resp_prefixes = resp['CommonPrefixes'] + self.assertEqual( + resp_prefixes, + [{'Prefix': p} for p in expect_prefixes]) def test_get_bucket_with_encoding_type(self): bucket = 'bucket' @@ -255,12 +241,10 @@ class TestS3ApiBucket(S3ApiBase): self._prepare_test_get_bucket(bucket, put_objects) encoding_type = 'url' - query = 'encoding-type=%s' % encoding_type - status, headers, body = \ - self.conn.make_request('GET', bucket, query=query) - self.assertEqual(status, 200) - elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('EncodingType').text, encoding_type) + resp = self.conn.list_objects( + Bucket=bucket, EncodingType=encoding_type) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(resp['EncodingType'], encoding_type) def test_get_bucket_with_marker(self): bucket = 'bucket' @@ -269,27 +253,11 @@ class TestS3ApiBucket(S3ApiBase): self._prepare_test_get_bucket(bucket, put_objects) marker = 'object' - query = 'marker=%s' % marker expect_objects = ('object2', 'subdir/object', 'subdir2/object') - status, headers, body = \ - self.conn.make_request('GET', bucket, query=query) - self.assertEqual(status, 200) - elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('Marker').text, marker) - resp_objects = elem.findall('./Contents') - self.assertEqual(len(list(resp_objects)), len(expect_objects)) - for i, o in enumerate(resp_objects): - self.assertEqual(o.find('Key').text, expect_objects[i]) - self.assertIsNotNone(o.find('LastModified').text) - self.assertRegexpMatches( - o.find('LastModified').text, - r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') - self.assertIsNotNone(o.find('ETag').text) - self.assertIsNotNone(o.find('Size').text) - self.assertEqual(o.find('StorageClass').text, 'STANDARD') - self.assertEqual(o.find('Owner/ID').text, self.conn.user_id) - self.assertEqual(o.find('Owner/DisplayName').text, - self.conn.user_id) + resp = self.conn.list_objects(Bucket=bucket, Marker=marker) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(resp['Marker'], marker) + self._validate_object_listing(resp['Contents'], expect_objects) def test_get_bucket_with_max_keys(self): bucket = 'bucket' @@ -297,28 +265,12 @@ class TestS3ApiBucket(S3ApiBase): 'dir/subdir/object') self._prepare_test_get_bucket(bucket, put_objects) - max_keys = '2' - query = 'max-keys=%s' % max_keys + max_keys = 2 expect_objects = ('dir/subdir/object', 'object') - status, headers, body = \ - self.conn.make_request('GET', bucket, query=query) - self.assertEqual(status, 200) - elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('MaxKeys').text, max_keys) - resp_objects = elem.findall('./Contents') - self.assertEqual(len(list(resp_objects)), len(expect_objects)) - for i, o in enumerate(resp_objects): - self.assertEqual(o.find('Key').text, expect_objects[i]) - self.assertIsNotNone(o.find('LastModified').text) - self.assertRegexpMatches( - o.find('LastModified').text, - r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') - self.assertIsNotNone(o.find('ETag').text) - self.assertIsNotNone(o.find('Size').text) - self.assertEqual(o.find('StorageClass').text, 'STANDARD') - self.assertEqual(o.find('Owner/ID').text, self.conn.user_id) - self.assertEqual(o.find('Owner/DisplayName').text, - self.conn.user_id) + resp = self.conn.list_objects(Bucket=bucket, MaxKeys=max_keys) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(resp['MaxKeys'], max_keys) + self._validate_object_listing(resp['Contents'], expect_objects) def test_get_bucket_with_prefix(self): bucket = 'bucket' @@ -327,27 +279,11 @@ class TestS3ApiBucket(S3ApiBase): self._prepare_test_get_bucket(bucket, req_objects) prefix = 'object' - query = 'prefix=%s' % prefix expect_objects = ('object', 'object2') - status, headers, body = \ - self.conn.make_request('GET', bucket, query=query) - self.assertEqual(status, 200) - elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('Prefix').text, prefix) - resp_objects = elem.findall('./Contents') - self.assertEqual(len(list(resp_objects)), len(expect_objects)) - for i, o in enumerate(resp_objects): - self.assertEqual(o.find('Key').text, expect_objects[i]) - self.assertIsNotNone(o.find('LastModified').text) - self.assertRegexpMatches( - o.find('LastModified').text, - r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') - self.assertIsNotNone(o.find('ETag').text) - self.assertIsNotNone(o.find('Size').text) - self.assertEqual(o.find('StorageClass').text, 'STANDARD') - self.assertEqual(o.find('Owner/ID').text, self.conn.user_id) - self.assertEqual(o.find('Owner/DisplayName').text, - self.conn.user_id) + resp = self.conn.list_objects(Bucket=bucket, Prefix=prefix) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(resp['Prefix'], prefix) + self._validate_object_listing(resp['Contents'], expect_objects) def test_get_bucket_v2_with_start_after(self): bucket = 'bucket' @@ -356,26 +292,13 @@ class TestS3ApiBucket(S3ApiBase): self._prepare_test_get_bucket(bucket, put_objects) marker = 'object' - query = 'list-type=2&start-after=%s' % marker expect_objects = ('object2', 'subdir/object', 'subdir2/object') - status, headers, body = \ - self.conn.make_request('GET', bucket, query=query) - self.assertEqual(status, 200) - elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('StartAfter').text, marker) - resp_objects = elem.findall('./Contents') - self.assertEqual(len(list(resp_objects)), len(expect_objects)) - for i, o in enumerate(resp_objects): - self.assertEqual(o.find('Key').text, expect_objects[i]) - self.assertTrue(o.find('LastModified').text is not None) - self.assertRegexpMatches( - o.find('LastModified').text, - r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') - self.assertTrue(o.find('ETag').text is not None) - self.assertTrue(o.find('Size').text is not None) - self.assertEqual(o.find('StorageClass').text, 'STANDARD') - self.assertIsNone(o.find('Owner/ID')) - self.assertIsNone(o.find('Owner/DisplayName')) + resp = self.conn.list_objects_v2(Bucket=bucket, StartAfter=marker) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(resp['StartAfter'], marker) + self.assertEqual(resp['KeyCount'], 3) + self._validate_object_listing(resp['Contents'], expect_objects, + expect_owner=False) def test_get_bucket_v2_with_fetch_owner(self): bucket = 'bucket' @@ -383,28 +306,12 @@ class TestS3ApiBucket(S3ApiBase): 'dir/subdir/object') self._prepare_test_get_bucket(bucket, put_objects) - query = 'list-type=2&fetch-owner=true' expect_objects = ('dir/subdir/object', 'object', 'object2', 'subdir/object', 'subdir2/object') - status, headers, body = \ - self.conn.make_request('GET', bucket, query=query) - self.assertEqual(status, 200) - elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('KeyCount').text, '5') - resp_objects = elem.findall('./Contents') - self.assertEqual(len(list(resp_objects)), len(expect_objects)) - for i, o in enumerate(resp_objects): - self.assertEqual(o.find('Key').text, expect_objects[i]) - self.assertTrue(o.find('LastModified').text is not None) - self.assertRegexpMatches( - o.find('LastModified').text, - r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') - self.assertTrue(o.find('ETag').text is not None) - self.assertTrue(o.find('Size').text is not None) - self.assertEqual(o.find('StorageClass').text, 'STANDARD') - self.assertTrue(o.find('Owner/ID').text, self.conn.user_id) - self.assertTrue(o.find('Owner/DisplayName').text, - self.conn.user_id) + resp = self.conn.list_objects_v2(Bucket=bucket, FetchOwner=True) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(resp['KeyCount'], 5) + self._validate_object_listing(resp['Contents'], expect_objects) def test_get_bucket_v2_with_continuation_token_and_delimiter(self): bucket = 'bucket' @@ -421,86 +328,116 @@ class TestS3ApiBucket(S3ApiBase): 'subdirs': []}] continuation_token = '' - query = 'list-type=2&max-keys=3&delimiter=/&continuation-token=%s' for i in range(len(expected)): - status, headers, body = self.conn.make_request( - 'GET', bucket, query=query % continuation_token) - self.assertEqual(status, 200) - elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('MaxKeys').text, '3') + resp = self.conn.list_objects_v2( + Bucket=bucket, + MaxKeys=3, + Delimiter='/', + ContinuationToken=continuation_token) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(resp['MaxKeys'], 3) self.assertEqual( - elem.find('KeyCount').text, - str(len(expected[i]['objects']) + len(expected[i]['subdirs']))) - expect_truncated = 'true' if i < len(expected) - 1 else 'false' - self.assertEqual(elem.find('IsTruncated').text, expect_truncated) - next_cont_token_elem = elem.find('NextContinuationToken') - if expect_truncated == 'true': - self.assertIsNotNone(next_cont_token_elem) - continuation_token = next_cont_token_elem.text - resp_objects = elem.findall('./Contents') + resp['KeyCount'], + len(expected[i]['objects']) + len(expected[i]['subdirs'])) + expect_truncated = i < len(expected) - 1 + self.assertEqual(resp['IsTruncated'], expect_truncated) + if expect_truncated: + self.assertIsNotNone(resp['NextContinuationToken']) + continuation_token = resp['NextContinuationToken'] + self._validate_object_listing(resp['Contents'], + expected[i]['objects'], + expect_owner=False) + resp_subdirs = resp.get('CommonPrefixes', []) self.assertEqual( - len(list(resp_objects)), len(expected[i]['objects'])) - for j, o in enumerate(resp_objects): - self.assertEqual(o.find('Key').text, - expected[i]['objects'][j].encode('utf-8')) - self.assertTrue(o.find('LastModified').text is not None) - self.assertRegexpMatches( - o.find('LastModified').text, - r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') - self.assertTrue(o.find('ETag').text is not None) - self.assertTrue(o.find('Size').text is not None) - self.assertEqual(o.find('StorageClass').text, 'STANDARD') - self.assertIsNone(o.find('Owner/ID')) - self.assertIsNone(o.find('Owner/DisplayName')) - resp_subdirs = elem.findall('./CommonPrefixes') - self.assertEqual( - len(list(resp_subdirs)), len(expected[i]['subdirs'])) - for j, o in enumerate(resp_subdirs): - self.assertEqual( - o.find('Prefix').text, - expected[i]['subdirs'][j].encode('utf-8')) + resp_subdirs, + [{'Prefix': p} for p in expected[i]['subdirs']]) def test_head_bucket_error(self): - self.conn.make_request('PUT', 'bucket') + event_system = self.conn.meta.events + event_system.unregister( + 'before-parameter-build.s3', + botocore.handlers.validate_bucket_name) - status, headers, body = \ - self.conn.make_request('HEAD', 'bucket+invalid') - self.assertEqual(status, 400) - self.assertEqual(body, '') # sanity + self.conn.create_bucket(Bucket='bucket') - auth_error_conn = Connection(aws_secret_key='invalid') - status, headers, body = \ - auth_error_conn.make_request('HEAD', 'bucket') - self.assertEqual(status, 403) - self.assertEqual(body, '') # sanity + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + self.conn.head_bucket(Bucket='bucket+invalid') + self.assertEqual( + ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 400) + self.assertEqual(ctx.exception.response['Error']['Code'], '400') + self.assertEqual( + ctx.exception.response[ + 'ResponseMetadata']['HTTPHeaders']['content-length'], '0') - status, headers, body = self.conn.make_request('HEAD', 'nothing') - self.assertEqual(status, 404) - self.assertEqual(body, '') # sanity + auth_error_conn = get_boto3_conn(aws_secret_key='invalid') + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + auth_error_conn.head_bucket(Bucket='bucket') + self.assertEqual( + ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 403) + self.assertEqual( + ctx.exception.response['Error']['Code'], '403') + self.assertEqual( + ctx.exception.response[ + 'ResponseMetadata']['HTTPHeaders']['content-length'], '0') + + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + self.conn.head_bucket(Bucket='nothing') + self.assertEqual( + ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 404) + self.assertEqual( + ctx.exception.response['Error']['Code'], '404') + self.assertEqual( + ctx.exception.response[ + 'ResponseMetadata']['HTTPHeaders']['content-length'], '0') def test_delete_bucket_error(self): - status, headers, body = \ - self.conn.make_request('DELETE', 'bucket+invalid') - self.assertEqual(get_error_code(body), 'InvalidBucketName') + event_system = self.conn.meta.events + event_system.unregister( + 'before-parameter-build.s3', + botocore.handlers.validate_bucket_name) + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + self.conn.delete_bucket(Bucket='bucket+invalid') + self.assertEqual( + ctx.exception.response['Error']['Code'], 'InvalidBucketName') - auth_error_conn = Connection(aws_secret_key='invalid') - status, headers, body = \ - auth_error_conn.make_request('DELETE', 'bucket') - self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch') + auth_error_conn = get_boto3_conn(aws_secret_key='invalid') + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + auth_error_conn.delete_bucket(Bucket='bucket') + self.assertEqual( + ctx.exception.response['Error']['Code'], 'SignatureDoesNotMatch') - status, headers, body = self.conn.make_request('DELETE', 'bucket') - self.assertEqual(get_error_code(body), 'NoSuchBucket') + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + self.conn.delete_bucket(Bucket='bucket') + self.assertEqual( + ctx.exception.response['Error']['Code'], 'NoSuchBucket') def test_bucket_invalid_method_error(self): + def _mangle_req_method(request, **kwargs): + request.method = 'GETPUT' + + def _mangle_req_controller_method(request, **kwargs): + request.method = '_delete_segments_bucket' + + event_system = self.conn.meta.events + event_system.register( + 'request-created.s3.CreateBucket', + _mangle_req_method) # non existed verb in the controller - status, headers, body = \ - self.conn.make_request('GETPUT', 'bucket') - self.assertEqual(get_error_code(body), 'MethodNotAllowed') + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + self.conn.create_bucket(Bucket='bucket') + self.assertEqual( + ctx.exception.response['Error']['Code'], 'MethodNotAllowed') + + event_system.unregister('request-created.s3.CreateBucket', + _mangle_req_method) + event_system.register('request-created.s3.CreateBucket', + _mangle_req_controller_method) # the method exists in the controller but deny as MethodNotAllowed - status, headers, body = \ - self.conn.make_request('_delete_segments_bucket', 'bucket') - self.assertEqual(get_error_code(body), 'MethodNotAllowed') + with self.assertRaises(botocore.exceptions.ClientError) as ctx: + self.conn.create_bucket(Bucket='bucket') + self.assertEqual( + ctx.exception.response['Error']['Code'], 'MethodNotAllowed') class TestS3ApiBucketSigV4(TestS3ApiBucket):