openstack/swift
Code Issues Proposed changes

Merge "Remove swift3 from here."

Browse Source
This commit is contained in:
Jenkins 2012-05-24 15:30:17 +00:00 committed by Gerrit Code Review
commit a74cd3b01b
5 changed files with 6 additions and 1099 deletions
doc/source
associated_projects.rstmisc.rst
setup.py
swift/common/middleware
swift3.py
test/unit/common/middleware
test_swift3.py

6
doc/source/associated_projects.rst

@ -49,6 +49,12 @@ Content Distribution Network Integration
* `SOS <https://github.com/dpgoetz/sos>`_ - Swift Origin Server.
Alternative API
---------------
* `Swift3 <https://github.com/fujita/swift3>`_ - Amazon S3 API emulation.
Other
-----

7
doc/source/misc.rst

@ -140,13 +140,6 @@ Ratelimit
:members:
:show-inheritance:
Swift3
======
.. automodule:: swift.common.middleware.swift3
:members:
:show-inheritance:
StaticWeb
=========

1
setup.py

@ -87,7 +87,6 @@ setup(
'cname_lookup=swift.common.middleware.cname_lookup:filter_factory',
'catch_errors=swift.common.middleware.catch_errors:filter_factory',
'domain_remap=swift.common.middleware.domain_remap:filter_factory',
'swift3=swift.common.middleware.swift3:filter_factory',
'staticweb=swift.common.middleware.staticweb:filter_factory',
'tempauth=swift.common.middleware.tempauth:filter_factory',
'recon=swift.common.middleware.recon:filter_factory',

480
swift/common/middleware/swift3.py

@ -1,480 +0,0 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
The swift3 middleware will emulate the S3 REST api on top of swift.
The following opperations are currently supported:
* GET Service
* DELETE Bucket
* GET Bucket (List Objects)
* PUT Bucket
* DELETE Object
* GET Object
* HEAD Object
* PUT Object
* PUT Object (Copy)
To add this middleware to your configuration, add the swift3 middleware
in front of the auth middleware, and before any other middleware that
look at swift requests (like rate limiting).
To set up your client, the access key will be the concatenation of the
account and user strings that should look like test:tester, and the
secret access key is the account password. The host should also point
to the swift storage hostname. It also will have to use the old style
calling format, and not the hostname based container format.
An example client using the python boto library might look like the
following for an SAIO setup::
connection = boto.s3.Connection(
aws_access_key_id='test:tester',
aws_secret_access_key='testing',
port=8080,
host='127.0.0.1',
is_secure=False,
calling_format=boto.s3.connection.OrdinaryCallingFormat())
"""
from urllib import unquote, quote
import base64
from xml.sax.saxutils import escape as xml_escape
import urlparse
from webob import Request, Response
from simplejson import loads
from swift.common.utils import split_path
from swift.common.wsgi import WSGIContext
from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
HTTP_NO_CONTENT, HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, \
HTTP_NOT_FOUND, HTTP_CONFLICT, is_success
MAX_BUCKET_LISTING = 1000
def get_err_response(code):
"""
Given an HTTP response code, create a properly formatted xml error response
:param code: error code
:returns: webob.response object
"""
error_table = {
'AccessDenied':
(HTTP_FORBIDDEN, 'Access denied'),
'BucketAlreadyExists':
(HTTP_CONFLICT, 'The requested bucket name is not available'),
'BucketNotEmpty':
(HTTP_CONFLICT, 'The bucket you tried to delete is not empty'),
'InvalidArgument':
(HTTP_BAD_REQUEST, 'Invalid Argument'),
'InvalidBucketName':
(HTTP_BAD_REQUEST, 'The specified bucket is not valid'),
'InvalidURI':
(HTTP_BAD_REQUEST, 'Could not parse the specified URI'),
'NoSuchBucket':
(HTTP_NOT_FOUND, 'The specified bucket does not exist'),
'SignatureDoesNotMatch':
(HTTP_FORBIDDEN, 'The calculated request signature does not '\
'match your provided one'),
'NoSuchKey':
(HTTP_NOT_FOUND, 'The resource you requested does not exist')}
resp = Response(content_type='text/xml')
resp.status = error_table[code][0]
resp.body = error_table[code][1]
resp.body = '<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n ' \
'<Code>%s</Code>\r\n <Message>%s</Message>\r\n</Error>\r\n' \
% (code, error_table[code][1])
return resp
def get_acl(account_name):
body = ('<AccessControlPolicy>'
'<Owner>'
'<ID>%s</ID>'
'</Owner>'
'<AccessControlList>'
'<Grant>'
'<Grantee xmlns:xsi="http://www.w3.org/2001/'\
'XMLSchema-instance" xsi:type="CanonicalUser">'
'<ID>%s</ID>'
'</Grantee>'
'<Permission>FULL_CONTROL</Permission>'
'</Grant>'
'</AccessControlList>'
'</AccessControlPolicy>' %
(account_name, account_name))
return Response(body=body, content_type="text/plain")
def canonical_string(req):
"""
Canonicalize a request to a token that can be signed.
"""
amz_headers = {}
buf = "%s\n%s\n%s\n" % (req.method, req.headers.get('Content-MD5', ''),
req.headers.get('Content-Type') or '')
for amz_header in sorted((key.lower() for key in req.headers
if key.lower().startswith('x-amz-'))):
amz_headers[amz_header] = req.headers[amz_header]
if 'x-amz-date' in amz_headers:
buf += "\n"
elif 'Date' in req.headers:
buf += "%s\n" % req.headers['Date']
for k in sorted(key.lower() for key in amz_headers):
buf += "%s:%s\n" % (k, amz_headers[k])
path = req.path_qs
if '?' in path:
path, args = path.split('?', 1)
for key in urlparse.parse_qs(args, keep_blank_values=True):
if key in ('acl', 'logging', 'torrent', 'location',
'requestPayment'):
return "%s%s?%s" % (buf, path, key)
return buf + path
class ServiceController(WSGIContext):
"""
Handles account level requests.
"""
def __init__(self, env, app, account_name, token, **kwargs):
WSGIContext.__init__(self, app)
env['HTTP_X_AUTH_TOKEN'] = token
env['PATH_INFO'] = '/v1/%s' % account_name
def GET(self, env, start_response):
"""
Handle GET Service request
"""
env['QUERY_STRING'] = 'format=json'
body_iter = self._app_call(env)
status = self._get_status_int()
if status != HTTP_OK:
if status == HTTP_UNAUTHORIZED:
return get_err_response('AccessDenied')
else:
return get_err_response('InvalidURI')
containers = loads(''.join(list(body_iter)))
# we don't keep the creation time of a backet (s3cmd doesn't
# work without that) so we use something bogus.
body = '<?xml version="1.0" encoding="UTF-8"?>' \
'<ListAllMyBucketsResult ' \
'xmlns="http://doc.s3.amazonaws.com/2006-03-01">' \
'<Buckets>%s</Buckets>' \
'</ListAllMyBucketsResult>' \
% ("".join(['<Bucket><Name>%s</Name><CreationDate>' \
'2009-02-03T16:45:09.000Z</CreationDate></Bucket>' %
xml_escape(i['name']) for i in containers]))
resp = Response(status=HTTP_OK, content_type='application/xml',
body=body)
return resp
class BucketController(WSGIContext):
"""
Handles bucket request.
"""
def __init__(self, env, app, account_name, token, container_name,
**kwargs):
WSGIContext.__init__(self, app)
self.container_name = unquote(container_name)
self.account_name = unquote(account_name)
env['HTTP_X_AUTH_TOKEN'] = token
env['PATH_INFO'] = '/v1/%s/%s' % (account_name, container_name)
def GET(self, env, start_response):
"""
Handle GET Bucket (List Objects) request
"""
if 'QUERY_STRING' in env:
args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1))
else:
args = {}
max_keys = min(int(args.get('max-keys', MAX_BUCKET_LISTING)),
MAX_BUCKET_LISTING)
env['QUERY_STRING'] = 'format=json&limit=%s' % (max_keys + 1)
if 'marker' in args:
env['QUERY_STRING'] += '&marker=%s' % quote(args['marker'])
if 'prefix' in args:
env['QUERY_STRING'] += '&prefix=%s' % quote(args['prefix'])
if 'delimiter' in args:
env['QUERY_STRING'] += '&delimiter=%s' % quote(args['delimiter'])
body_iter = self._app_call(env)
status = self._get_status_int()
if status != HTTP_OK:
if status == HTTP_UNAUTHORIZED:
return get_err_response('AccessDenied')
elif status == HTTP_NOT_FOUND:
return get_err_response('NoSuchBucket')
else:
return get_err_response('InvalidURI')
if 'acl' in args:
return get_acl(self.account_name)
objects = loads(''.join(list(body_iter)))
body = ('<?xml version="1.0" encoding="UTF-8"?>'
'<ListBucketResult '
'xmlns="http://s3.amazonaws.com/doc/2006-03-01">'
'<Prefix>%s</Prefix>'
'<Marker>%s</Marker>'
'<Delimiter>%s</Delimiter>'
'<IsTruncated>%s</IsTruncated>'
'<MaxKeys>%s</MaxKeys>'
'<Name>%s</Name>'
'%s'
'%s'
'</ListBucketResult>' %
(
xml_escape(args.get('prefix', '')),
xml_escape(args.get('marker', '')),
xml_escape(args.get('delimiter', '')),
'true' if len(objects) == (max_keys + 1) else 'false',
max_keys,
xml_escape(self.container_name),
"".join(['<Contents><Key>%s</Key><LastModified>%sZ</LastModif'\
'ied><ETag>%s</ETag><Size>%s</Size><StorageClass>STA'\
'NDARD</StorageClass></Contents>' %
(xml_escape(i['name']), i['last_modified'], i['hash'],
i['bytes'])
for i in objects[:max_keys] if 'subdir' not in i]),
"".join(['<CommonPrefixes><Prefix>%s</Prefix></CommonPrefixes>'
% xml_escape(i['subdir'])
for i in objects[:max_keys] if 'subdir' in i])))
return Response(body=body, content_type='application/xml')
def PUT(self, env, start_response):
"""
Handle PUT Bucket request
"""
body_iter = self._app_call(env)
status = self._get_status_int()
if status != HTTP_CREATED:
if status == HTTP_UNAUTHORIZED:
return get_err_response('AccessDenied')
elif status == HTTP_ACCEPTED:
return get_err_response('BucketAlreadyExists')
else:
return get_err_response('InvalidURI')
resp = Response()
resp.headers.add('Location', self.container_name)
resp.status = HTTP_OK
return resp
def DELETE(self, env, start_response):
"""
Handle DELETE Bucket request
"""
body_iter = self._app_call(env)
status = self._get_status_int()
if status != HTTP_NO_CONTENT:
if status == HTTP_UNAUTHORIZED:
return get_err_response('AccessDenied')
elif status == HTTP_NOT_FOUND:
return get_err_response('NoSuchBucket')
elif status == HTTP_CONFLICT:
return get_err_response('BucketNotEmpty')
else:
return get_err_response('InvalidURI')
resp = Response()
resp.status = HTTP_NO_CONTENT
return resp
class ObjectController(WSGIContext):
"""
Handles requests on objects
"""
def __init__(self, env, app, account_name, token, container_name,
object_name, **kwargs):
WSGIContext.__init__(self, app)
self.account_name = unquote(account_name)
self.container_name = unquote(container_name)
env['HTTP_X_AUTH_TOKEN'] = token
env['PATH_INFO'] = '/v1/%s/%s/%s' % (account_name, container_name,
object_name)
def GETorHEAD(self, env, start_response):
app_iter = self._app_call(env)
status = self._get_status_int()
headers = dict(self._response_headers)
if is_success(status):
if 'QUERY_STRING' in env:
args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1))
else:
args = {}
if 'acl' in args:
return get_acl(self.account_name)
new_hdrs = {}
for key, val in headers.iteritems():
_key = key.lower()
if _key.startswith('x-object-meta-'):
new_hdrs['x-amz-meta-' + key[14:]] = val
elif _key in ('content-length', 'content-type',
'content-range', 'content-encoding',
'etag', 'last-modified'):
new_hdrs[key] = val
return Response(status=status, headers=new_hdrs, app_iter=app_iter)
elif status == HTTP_UNAUTHORIZED:
return get_err_response('AccessDenied')
elif status == HTTP_NOT_FOUND:
return get_err_response('NoSuchKey')
else:
return get_err_response('InvalidURI')
def HEAD(self, env, start_response):
"""
Handle HEAD Object request
"""
return self.GETorHEAD(env, start_response)
def GET(self, env, start_response):
"""
Handle GET Object request
"""
return self.GETorHEAD(env, start_response)
def PUT(self, env, start_response):
"""
Handle PUT Object and PUT Object (Copy) request
"""
for key, value in env.items():
if key.startswith('HTTP_X_AMZ_META_'):
del env[key]
env['HTTP_X_OBJECT_META_' + key[16:]] = value
elif key == 'HTTP_CONTENT_MD5':
env['HTTP_ETAG'] = value.decode('base64').encode('hex')
elif key == 'HTTP_X_AMZ_COPY_SOURCE':
env['HTTP_X_COPY_FROM'] = value
body_iter = self._app_call(env)
status = self._get_status_int()
if status != HTTP_CREATED:
if status == HTTP_UNAUTHORIZED:
return get_err_response('AccessDenied')
elif status == HTTP_NOT_FOUND:
return get_err_response('NoSuchBucket')
else:
return get_err_response('InvalidURI')
if 'HTTP_X_COPY_FROM' in env:
body = '<CopyObjectResult>' \
'<ETag>"%s"</ETag>' \
'</CopyObjectResult>' % self._response_header_value('etag')
return Response(status=HTTP_OK, body=body)
return Response(status=200, etag=self._response_header_value('etag'))
def DELETE(self, env, start_response):
"""
Handle DELETE Object request
"""
body_iter = self._app_call(env)
status = self._get_status_int()
if status != HTTP_NO_CONTENT:
if status == HTTP_UNAUTHORIZED:
return get_err_response('AccessDenied')
elif status == HTTP_NOT_FOUND:
return get_err_response('NoSuchKey')
else:
return get_err_response('InvalidURI')
resp = Response()
resp.status = HTTP_NO_CONTENT
return resp
class Swift3Middleware(object):
"""Swift3 S3 compatibility midleware"""
def __init__(self, app, conf, *args, **kwargs):
self.app = app
def get_controller(self, path):
container, obj = split_path(path, 0, 2, True)
d = dict(container_name=container, object_name=obj)
if container and obj:
return ObjectController, d
elif container:
return BucketController, d
return ServiceController, d
def __call__(self, env, start_response):
req = Request(env)
if 'AWSAccessKeyId' in req.GET:
try:
req.headers['Date'] = req.GET['Expires']
req.headers['Authorization'] = \
'AWS %(AWSAccessKeyId)s:%(Signature)s' % req.GET
except KeyError:
return get_err_response('InvalidArgument')(env, start_response)
if not 'Authorization' in req.headers:
return self.app(env, start_response)
try:
account, signature = \
req.headers['Authorization'].split(' ')[-1].rsplit(':', 1)
except Exception:
return get_err_response('InvalidArgument')(env, start_response)
try:
controller, path_parts = self.get_controller(req.path)
except ValueError:
return get_err_response('InvalidURI')(env, start_response)
token = base64.urlsafe_b64encode(canonical_string(req))
controller = controller(env, self.app, account, token, **path_parts)
if hasattr(controller, req.method):
res = getattr(controller, req.method)(env, start_response)
else:
return get_err_response('InvalidURI')(env, start_response)
return res(env, start_response)
def filter_factory(global_conf, **local_conf):
"""Standard filter factory to use the middleware with paste.deploy"""
conf = global_conf.copy()
conf.update(local_conf)
def swift3_filter(app):
return Swift3Middleware(app, conf)
return swift3_filter

611
test/unit/common/middleware/test_swift3.py

@ -1,611 +0,0 @@
# Copyright (c) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from datetime import datetime
import cgi
import hashlib
from webob import Request, Response
from webob.exc import HTTPUnauthorized, HTTPCreated, HTTPNoContent,\
HTTPAccepted, HTTPBadRequest, HTTPNotFound, HTTPConflict
import xml.dom.minidom
import simplejson
from swift.common.middleware import swift3
class FakeApp(object):
def __init__(self):
self.app = self
self.response_args = []
def __call__(self, env, start_response):
return "FAKE APP"
def do_start_response(self, *args):
self.response_args.extend(args)
class FakeAppService(FakeApp):
def __init__(self, status=200):
FakeApp.__init__(self)
self.status = status
self.buckets = (('apple', 1, 200), ('orange', 3, 430))
def __call__(self, env, start_response):
if self.status == 200:
start_response(Response().status, [('Content-Type', 'text/xml')])
json_pattern = ['"name":%s', '"count":%s', '"bytes":%s']
json_pattern = '{' + ','.join(json_pattern) + '}'
json_out = []
for b in self.buckets:
name = simplejson.dumps(b[0])
json_out.append(json_pattern %
(name, b[1], b[2]))
account_list = '[' + ','.join(json_out) + ']'
return account_list
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
else:
start_response(HTTPBadRequest().status, [])
return []
class FakeAppBucket(FakeApp):
def __init__(self, status=200):
FakeApp.__init__(self)
self.status = status
self.objects = (('rose', '2011-01-05T02:19:14.275290', 0, 303),
('viola', '2011-01-05T02:19:14.275290', 0, 3909),
('lily', '2011-01-05T02:19:14.275290', 0, 3909))
def __call__(self, env, start_response):
if env['REQUEST_METHOD'] == 'GET':
if self.status == 200:
start_response(Response().status,
[('Content-Type', 'text/xml')])
json_pattern = ['"name":%s', '"last_modified":%s', '"hash":%s',
'"bytes":%s']
json_pattern = '{' + ','.join(json_pattern) + '}'
json_out = []
for b in self.objects:
name = simplejson.dumps(b[0])
time = simplejson.dumps(b[1])
json_out.append(json_pattern %
(name, time, b[2], b[3]))
account_list = '[' + ','.join(json_out) + ']'
return account_list
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'PUT':
if self.status == 201:
start_response(HTTPCreated().status, [])
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 202:
start_response(HTTPAccepted().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'DELETE':
if self.status == 204:
start_response(HTTPNoContent().status, [])
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
elif self.status == 409:
start_response(HTTPConflict().status, [])
else:
start_response(HTTPBadRequest().status, [])
return []
class FakeAppObject(FakeApp):
def __init__(self, status=200):
FakeApp.__init__(self)
self.status = status
self.object_body = 'hello'
self.response_headers = {'Content-Type': 'text/html',
'Content-Length': len(self.object_body),
'x-object-meta-test': 'swift',
'etag': '1b2cf535f27731c974343645a3985328',
'last-modified': '2011-01-05T02:19:14.275290'}
def __call__(self, env, start_response):
if env['REQUEST_METHOD'] == 'GET' or env['REQUEST_METHOD'] == 'HEAD':
if self.status == 200:
if 'HTTP_RANGE' in env:
resp = Response(body=self.object_body,
conditional_response=True)
return resp(env, start_response)
start_response(Response().status,
self.response_headers.items())
if env['REQUEST_METHOD'] == 'GET':
return self.object_body
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'PUT':
if self.status == 201:
start_response(HTTPCreated().status,
[('etag', self.response_headers['etag'])])
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'DELETE':
if self.status == 204:
start_response(HTTPNoContent().status, [])
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
else:
start_response(HTTPBadRequest().status, [])
return []
def start_response(*args):
pass
class TestSwift3(unittest.TestCase):
def setUp(self):
self.app = swift3.filter_factory({})(FakeApp())
def test_non_s3_request_passthrough(self):
req = Request.blank('/something')
resp = self.app(req.environ, start_response)
self.assertEquals(resp, 'FAKE APP')
def test_bad_format_authorization(self):
req = Request.blank('/something',
headers={'Authorization': 'hoge'})
resp = self.app(req.environ, start_response)
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.firstChild.nodeName, 'Error')
code = dom.getElementsByTagName('Code')[0].childNodes[0].nodeValue
self.assertEquals(code, 'InvalidArgument')
def test_bad_method(self):
req = Request.blank('/',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = self.app(req.environ, start_response)
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.firstChild.nodeName, 'Error')
code = dom.getElementsByTagName('Code')[0].childNodes[0].nodeValue
self.assertEquals(code, 'InvalidURI')
def _test_method_error(self, cl, method, path, status):
local_app = swift3.filter_factory({})(cl(status))
req = Request.blank(path,
environ={'REQUEST_METHOD': method},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, start_response)
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.firstChild.nodeName, 'Error')
return dom.getElementsByTagName('Code')[0].childNodes[0].nodeValue
def test_service_GET_error(self):
code = self._test_method_error(FakeAppService, 'GET', '/', 401)
self.assertEquals(code, 'AccessDenied')
code = self._test_method_error(FakeAppService, 'GET', '/', 0)
self.assertEquals(code, 'InvalidURI')
def test_service_GET(self):
local_app = swift3.filter_factory({})(FakeAppService())
req = Request.blank('/',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, local_app.app.do_start_response)
self.assertEquals(local_app.app.response_args[0].split()[0], '200')
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.firstChild.nodeName, 'ListAllMyBucketsResult')
buckets = [n for n in dom.getElementsByTagName('Bucket')]
listing = [n for n in buckets[0].childNodes if n.nodeName != '#text']
self.assertEquals(len(listing), 2)
names = []
for b in buckets:
if b.childNodes[0].nodeName == 'Name':
names.append(b.childNodes[0].childNodes[0].nodeValue)
self.assertEquals(len(names), len(FakeAppService().buckets))
for i in FakeAppService().buckets:
self.assertTrue(i[0] in names)
def test_bucket_GET_error(self):
code = self._test_method_error(FakeAppBucket, 'GET', '/bucket', 401)
self.assertEquals(code, 'AccessDenied')
code = self._test_method_error(FakeAppBucket, 'GET', '/bucket', 404)
self.assertEquals(code, 'NoSuchBucket')
code = self._test_method_error(FakeAppBucket, 'GET', '/bucket', 0)
self.assertEquals(code, 'InvalidURI')
def test_bucket_GET(self):
local_app = swift3.filter_factory({})(FakeAppBucket())
bucket_name = 'junk'
req = Request.blank('/%s' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, local_app.app.do_start_response)
self.assertEquals(local_app.app.response_args[0].split()[0], '200')
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.firstChild.nodeName, 'ListBucketResult')
name = dom.getElementsByTagName('Name')[0].childNodes[0].nodeValue
self.assertEquals(name, bucket_name)
objects = [n for n in dom.getElementsByTagName('Contents')]
listing = [n for n in objects[0].childNodes if n.nodeName != '#text']
names = []
for o in objects:
if o.childNodes[0].nodeName == 'Key':
names.append(o.childNodes[0].childNodes[0].nodeValue)
if o.childNodes[1].nodeName == 'LastModified':
self.assertTrue(
o.childNodes[1].childNodes[0].nodeValue.endswith('Z'))
self.assertEquals(len(names), len(FakeAppBucket().objects))
for i in FakeAppBucket().objects:
self.assertTrue(i[0] in names)
def test_bucket_GET_is_truncated(self):
local_app = swift3.filter_factory({})(FakeAppBucket())
bucket_name = 'junk'
req = Request.blank('/%s' % bucket_name,
environ={'REQUEST_METHOD': 'GET',
'QUERY_STRING': 'max-keys=3'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, local_app.app.do_start_response)
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.getElementsByTagName('IsTruncated')[0].
childNodes[0].nodeValue, 'false')
req = Request.blank('/%s' % bucket_name,
environ={'REQUEST_METHOD': 'GET',
'QUERY_STRING': 'max-keys=2'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, local_app.app.do_start_response)
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.getElementsByTagName('IsTruncated')[0].
childNodes[0].nodeValue, 'true')
def test_bucket_GET_max_keys(self):
class FakeApp(object):
def __call__(self, env, start_response):
self.query_string = env['QUERY_STRING']
start_response('200 OK', [])
return '[]'
fake_app = FakeApp()
local_app = swift3.filter_factory({})(fake_app)
bucket_name = 'junk'
req = Request.blank('/%s' % bucket_name,
environ={'REQUEST_METHOD': 'GET',
'QUERY_STRING': 'max-keys=5'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, lambda *args: None)
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.getElementsByTagName('MaxKeys')[0].
childNodes[0].nodeValue, '5')
args = dict(cgi.parse_qsl(fake_app.query_string))
self.assert_(args['limit'] == '6')
req = Request.blank('/%s' % bucket_name,
environ={'REQUEST_METHOD': 'GET',
'QUERY_STRING': 'max-keys=5000'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, lambda *args: None)
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.getElementsByTagName('MaxKeys')[0].
childNodes[0].nodeValue, '1000')
args = dict(cgi.parse_qsl(fake_app.query_string))
self.assertEquals(args['limit'], '1001')
def test_bucket_GET_passthroughs(self):
class FakeApp(object):
def __call__(self, env, start_response):
self.query_string = env['QUERY_STRING']
start_response('200 OK', [])
return '[]'
fake_app = FakeApp()
local_app = swift3.filter_factory({})(fake_app)
bucket_name = 'junk'
req = Request.blank('/%s' % bucket_name,
environ={'REQUEST_METHOD': 'GET', 'QUERY_STRING':
'delimiter=a&marker=b&prefix=c'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, lambda *args: None)
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.getElementsByTagName('Prefix')[0].
childNodes[0].nodeValue, 'c')
self.assertEquals(dom.getElementsByTagName('Marker')[0].
childNodes[0].nodeValue, 'b')
self.assertEquals(dom.getElementsByTagName('Delimiter')[0].
childNodes[0].nodeValue, 'a')
args = dict(cgi.parse_qsl(fake_app.query_string))
self.assertEquals(args['delimiter'], 'a')
self.assertEquals(args['marker'], 'b')
self.assertEquals(args['prefix'], 'c')
def test_bucket_PUT_error(self):
code = self._test_method_error(FakeAppBucket, 'PUT', '/bucket', 401)
self.assertEquals(code, 'AccessDenied')
code = self._test_method_error(FakeAppBucket, 'PUT', '/bucket', 202)
self.assertEquals(code, 'BucketAlreadyExists')
code = self._test_method_error(FakeAppBucket, 'PUT', '/bucket', 0)
self.assertEquals(code, 'InvalidURI')
def test_bucket_PUT(self):
local_app = swift3.filter_factory({})(FakeAppBucket(201))
req = Request.blank('/bucket',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, local_app.app.do_start_response)
self.assertEquals(local_app.app.response_args[0].split()[0], '200')
def test_bucket_DELETE_error(self):
code = self._test_method_error(FakeAppBucket, 'DELETE', '/bucket', 401)
self.assertEquals(code, 'AccessDenied')
code = self._test_method_error(FakeAppBucket, 'DELETE', '/bucket', 404)
self.assertEquals(code, 'NoSuchBucket')
code = self._test_method_error(FakeAppBucket, 'DELETE', '/bucket', 409)
self.assertEquals(code, 'BucketNotEmpty')
code = self._test_method_error(FakeAppBucket, 'DELETE', '/bucket', 0)
self.assertEquals(code, 'InvalidURI')
def test_bucket_DELETE(self):
local_app = swift3.filter_factory({})(FakeAppBucket(204))
req = Request.blank('/bucket',
environ={'REQUEST_METHOD': 'DELETE'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, local_app.app.do_start_response)
self.assertEquals(local_app.app.response_args[0].split()[0], '204')
def _check_acl(self, owner, resp):
dom = xml.dom.minidom.parseString("".join(resp))
self.assertEquals(dom.firstChild.nodeName, 'AccessControlPolicy')
name = dom.getElementsByTagName('Permission')[0].childNodes[0].nodeValue
self.assertEquals(name, 'FULL_CONTROL')
name = dom.getElementsByTagName('ID')[0].childNodes[0].nodeValue
self.assertEquals(name, owner)
def test_bucket_acl_GET(self):
local_app = swift3.filter_factory({})(FakeAppBucket())
bucket_name = 'junk'
req = Request.blank('/%s?acl' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, local_app.app.do_start_response)
self._check_acl('test:tester', resp)
def _test_object_GETorHEAD(self, method):
local_app = swift3.filter_factory({})(FakeAppObject())
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': method},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, local_app.app.do_start_response)
self.assertEquals(local_app.app.response_args[0].split()[0], '200')
headers = dict(local_app.app.response_args[1])
for key, val in local_app.app.response_headers.iteritems():
if key in ('Content-Length', 'Content-Type', 'Content-Encoding',
'etag', 'last-modified'):
self.assertTrue(key in headers)
self.assertEquals(headers[key], val)
elif key.startswith('x-object-meta-'):
self.assertTrue('x-amz-meta-' + key[14:] in headers)
self.assertEquals(headers['x-amz-meta-' + key[14:]], val)
if method == 'GET':
self.assertEquals(''.join(resp), local_app.app.object_body)
def test_object_HEAD(self):
self._test_object_GETorHEAD('HEAD')
def test_object_GET_error(self):
code = self._test_method_error(FakeAppObject, 'GET',
'/bucket/object', 401)
self.assertEquals(code, 'AccessDenied')
code = self._test_method_error(FakeAppObject, 'GET',
'/bucket/object', 404)
self.assertEquals(code, 'NoSuchKey')
code = self._test_method_error(FakeAppObject, 'GET',
'/bucket/object', 0)
self.assertEquals(code, 'InvalidURI')
def test_object_GET(self):
self._test_object_GETorHEAD('GET')
def test_object_GET_Range(self):
local_app = swift3.filter_factory({})(FakeAppObject())
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Range': 'bytes=0-3'})
resp = local_app(req.environ, local_app.app.do_start_response)
self.assertEquals(local_app.app.response_args[0].split()[0], '206')
headers = dict(local_app.app.response_args[1])
self.assertTrue('Content-Range' in headers)
self.assertTrue(headers['Content-Range'].startswith('bytes 0-3'))
def test_object_PUT_error(self):
code = self._test_method_error(FakeAppObject, 'PUT',
'/bucket/object', 401)
self.assertEquals(code, 'AccessDenied')
code = self._test_method_error(FakeAppObject, 'PUT',
'/bucket/object', 404)
self.assertEquals(code, 'NoSuchBucket')
code = self._test_method_error(FakeAppObject, 'PUT',
'/bucket/object', 0)
self.assertEquals(code, 'InvalidURI')
def test_object_PUT(self):
local_app = swift3.filter_factory({})(FakeAppObject(201))
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'x-amz-storage-class': 'REDUCED_REDUNDANCY',
'Content-MD5': 'Gyz1NfJ3Mcl0NDZFo5hTKA=='})
req.date = datetime.now()
req.content_type = 'text/plain'
resp = local_app(req.environ, local_app.app.do_start_response)
self.assertEquals(local_app.app.response_args[0].split()[0], '200')
headers = dict(local_app.app.response_args[1])
self.assertEquals(headers['ETag'],
"\"%s\"" % local_app.app.response_headers['etag'])
def test_object_PUT_headers(self):
class FakeApp(object):
def __call__(self, env, start_response):
self.req = Request(env)
start_response('200 OK', [])
return []
app = FakeApp()
local_app = swift3.filter_factory({})(app)
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'X-Amz-Storage-Class': 'REDUCED_REDUNDANCY',
'X-Amz-Meta-Something': 'oh hai',
'X-Amz-Copy-Source': '/some/source',
'Content-MD5': 'ffoHqOWd280dyE1MT4KuoQ=='})
req.date = datetime.now()
req.content_type = 'text/plain'
resp = local_app(req.environ, lambda *args: None)
self.assertEquals(app.req.headers['ETag'],
'7dfa07a8e59ddbcd1dc84d4c4f82aea1')
self.assertEquals(app.req.headers['X-Object-Meta-Something'], 'oh hai')
self.assertEquals(app.req.headers['X-Copy-From'], '/some/source')
def test_object_DELETE_error(self):
code = self._test_method_error(FakeAppObject, 'DELETE',
'/bucket/object', 401)
self.assertEquals(code, 'AccessDenied')
code = self._test_method_error(FakeAppObject, 'DELETE',
'/bucket/object', 404)
self.assertEquals(code, 'NoSuchKey')
code = self._test_method_error(FakeAppObject, 'DELETE',
'/bucket/object', 0)
self.assertEquals(code, 'InvalidURI')
def test_object_DELETE(self):
local_app = swift3.filter_factory({})(FakeAppObject(204))
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'DELETE'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, local_app.app.do_start_response)
self.assertEquals(local_app.app.response_args[0].split()[0], '204')
def test_object_acl_GET(self):
local_app = swift3.filter_factory({})(FakeAppObject())
req = Request.blank('/bucket/object?acl',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
resp = local_app(req.environ, local_app.app.do_start_response)
self._check_acl('test:tester', resp)
def test_canonical_string(self):
"""
The hashes here were generated by running the same requests against
boto.utils.canonical_string
"""
def verify(hash, path, headers):
req = Request.blank(path, headers=headers)
self.assertEquals(hash,
hashlib.md5(swift3.canonical_string(req)).hexdigest())
verify('6dd08c75e42190a1ce9468d1fd2eb787', '/bucket/object',
{'Content-Type': 'text/plain', 'X-Amz-Something': 'test',
'Date': 'whatever'})
verify('c8447135da232ae7517328f3429df481', '/bucket/object',
{'Content-Type': 'text/plain', 'X-Amz-Something': 'test'})
verify('bf49304103a4de5c325dce6384f2a4a2', '/bucket/object',
{'content-type': 'text/plain'})
verify('be01bd15d8d47f9fe5e2d9248cc6f180', '/bucket/object', {})
verify('8d28cc4b8322211f6cc003256cd9439e', 'bucket/object',
{'Content-MD5': 'somestuff'})
verify('a822deb31213ad09af37b5a7fe59e55e', '/bucket/object?acl', {})
verify('cce5dd1016595cb706c93f28d3eaa18f', '/bucket/object',
{'Content-Type': 'text/plain', 'X-Amz-A': 'test',
'X-Amz-Z': 'whatever', 'X-Amz-B': 'lalala',
'X-Amz-Y': 'lalalalalalala'})
verify('7506d97002c7d2de922cc0ec34af8846', '/bucket/object',
{'Content-Type': None, 'X-Amz-Something': 'test'})
verify('28f76d6162444a193b612cd6cb20e0be', '/bucket/object',
{'Content-Type': None,
'X-Amz-Date': 'Mon, 11 Jul 2011 10:52:57 +0000',
'Date': 'Tue, 12 Jul 2011 10:52:57 +0000'})
verify('ed6971e3eca5af4ee361f05d7c272e49', '/bucket/object',
{'Content-Type': None,
'Date': 'Tue, 12 Jul 2011 10:52:57 +0000'})
req1 = Request.blank('/', headers=
{'Content-Type': None, 'X-Amz-Something': 'test'})
req2 = Request.blank('/', headers=
{'Content-Type': '', 'X-Amz-Something': 'test'})
req3 = Request.blank('/', headers={'X-Amz-Something': 'test'})
self.assertEquals(swift3.canonical_string(req1),
swift3.canonical_string(req2))
self.assertEquals(swift3.canonical_string(req2),
swift3.canonical_string(req3))
def test_signed_urls(self):
class FakeApp(object):
def __call__(self, env, start_response):
self.req = Request(env)
start_response('200 OK', [])
return []
app = FakeApp()
local_app = swift3.filter_factory({})(app)
req = Request.blank('/bucket/object?Signature=X&Expires=Y&'
'AWSAccessKeyId=Z', environ={'REQUEST_METHOD': 'GET'})
req.date = datetime.now()
req.content_type = 'text/plain'
resp = local_app(req.environ, lambda *args: None)
self.assertEquals(app.req.headers['Authorization'], 'AWS Z:X')
self.assertEquals(app.req.headers['Date'], 'Y')
if __name__ == '__main__':
unittest.main()