openstack
/
swift
Code Issues Proposed changes

local WSGI Request and Response classes 

This change replaces WebOb with a mostly compatible local library,
swift.common.swob.  Subtle changes to WebOb's API over the years have been a
huge headache.  Swift doesn't even run on the current version.

There are a few incompatibilities to simplify the implementation/interface:
 * It only implements the header properties we use.  More can be easily added.
 * Casts header values to str on assignment.
 * Response classes ("HTTPNotFound") are no longer subclasses, but partials
   on Response, so things like isinstance no longer work on them.
 * Unlike newer webob versions, will never return unicode objects.

Change-Id: I76617a0903ee2286b25a821b3c935c86ff95233f
This commit is contained in:
Michael Barton 2012-09-04 14:02:19 -07:00
parent f0bd91dd14
commit 5e3e9a882d
54 changed files with 1448 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/source/debian_package_guide.rst
View File

@ -58,7 +58,7 @@ Instructions for Building Debian Packages for Swift
apt-get install python-software-properties
add-apt-repository ppa:swift-core/release
apt-get update
apt-get install curl gcc bzr python-configobj python-coverage python-dev python-nose python-setuptools python-simplejson python-xattr python-webob python-eventlet python-greenlet debhelper python-sphinx python-all python-openssl python-pastedeploy python-netifaces bzr-builddeb
apt-get install curl gcc bzr python-configobj python-coverage python-dev python-nose python-setuptools python-simplejson python-xattr python-eventlet python-greenlet debhelper python-sphinx python-all python-openssl python-pastedeploy python-netifaces bzr-builddeb
* As you
@ -105,7 +105,7 @@ Instructions for Deploying Debian Packages for Swift
#. Install dependencies::
apt-get install rsync python-openssl python-setuptools python-webob
apt-get install rsync python-openssl python-setuptools
python-simplejson python-xattr python-greenlet python-eventlet
python-netifaces

16
doc/source/development_auth.rst
View File

@ -63,7 +63,7 @@ Example Authentication with TempAuth:
Authorization is performed through callbacks by the Swift Proxy server to the
WSGI environment's swift.authorize value, if one is set. The swift.authorize
value should simply be a function that takes a webob.Request as an argument and
value should simply be a function that takes a Request as an argument and
returns None if access is granted or returns a callable(environ,
start_response) if access is denied. This callable is a standard WSGI callable.
Generally, you should return 403 Forbidden for requests by an authenticated
@ -71,7 +71,7 @@ user and 401 Unauthorized for an unauthenticated request. For example, here's
an authorize function that only allows GETs (in this case you'd probably return
405 Method Not Allowed, but ignore that for the moment).::
from webob.exc import HTTPForbidden, HTTPUnauthorized
from swift.common.swob import HTTPForbidden, HTTPUnauthorized
def authorize(req):
@ -87,7 +87,7 @@ middleware as authentication and authorization are often paired together. But,
you could create separate authorization middleware that simply sets the
callback before passing on the request. To continue our example above::
from webob.exc import HTTPForbidden, HTTPUnauthorized
from swift.common.swob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
@ -127,7 +127,7 @@ then swift.authorize will be called once more. These are called delay_denial
requests and currently include container read requests and object read and
write requests. For these requests, the read or write access control string
(X-Container-Read and X-Container-Write) will be fetched and set as the 'acl'
attribute in the webob.Request passed to swift.authorize.
attribute in the Request passed to swift.authorize.
The delay_denial procedures allow skipping possibly expensive access control
string retrievals for requests that can be approved without that information,
@ -138,7 +138,7 @@ control string set to same value as the authenticated user string. Note that
you probably wouldn't do this exactly as the access control string represents a
list rather than a single user, but it'll suffice for this example::
from webob.exc import HTTPForbidden, HTTPUnauthorized
from swift.common.swob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
@ -185,7 +185,7 @@ Let's continue our example to use parse_acl and referrer_allowed. Now we'll
only allow GETs after a referrer check and any requests after a group check::
from swift.common.middleware.acl import parse_acl, referrer_allowed
from webob.exc import HTTPForbidden, HTTPUnauthorized
from swift.common.swob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
@ -235,7 +235,7 @@ standard Swift format. Let's improve our example by making use of that::
from swift.common.middleware.acl import \
clean_acl, parse_acl, referrer_allowed
from webob.exc import HTTPForbidden, HTTPUnauthorized
from swift.common.swob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
@ -293,7 +293,7 @@ folks a start on their own code if they want to use repoze.what::
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
from swift.common.utils import cache_from_env, split_path
from webob.exc import HTTPForbidden, HTTPUnauthorized
from swift.common.swob import HTTPForbidden, HTTPUnauthorized
class DevAuthorization(object):

4
doc/source/development_saio.rst
View File

@ -30,8 +30,8 @@ Installing dependencies and the core code
#. `apt-get update`
#. `apt-get install curl gcc git-core memcached python-configobj
python-coverage python-dev python-nose python-setuptools python-simplejson
python-xattr sqlite3 xfsprogs python-webob python-eventlet
python-greenlet python-pastedeploy python-netifaces python-pip`
python-xattr sqlite3 xfsprogs python-eventlet python-greenlet
python-pastedeploy python-netifaces python-pip`
#. `pip install mock`
#. Install anything else you want, like screen, ssh, vim, etc.

1
doc/source/getting_started.rst
View File

@ -15,7 +15,6 @@ most Linux platforms with the following software:
And the following python libraries:
* Eventlet 0.9.8
* WebOb 0.9.8
* Setuptools
* Simplejson
* Xattr

11
swift/account/server.py
View File

@ -22,11 +22,6 @@ from urllib import unquote
from xml.sax import saxutils
from eventlet import Timeout
from webob import Request, Response
from webob.exc import HTTPAccepted, HTTPBadRequest, \
HTTPCreated, HTTPForbidden, HTTPInternalServerError, \
HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \
HTTPPreconditionFailed, HTTPConflict
import swift.common.db
from swift.common.db import AccountBroker
@ -36,7 +31,11 @@ from swift.common.utils import get_logger, get_param, hash_path, public, \
from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
check_mount, check_float, check_utf8, FORMAT2CONTENT_TYPE
from swift.common.db_replicator import ReplicatorRpc
from swift.common.http import HTTPInsufficientStorage
from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
HTTPCreated, HTTPForbidden, HTTPInternalServerError, \
HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \
HTTPPreconditionFailed, HTTPConflict, Request, Response, \
HTTPInsufficientStorage
DATADIR = 'accounts'

2
swift/common/constraints.py
View File

@ -17,7 +17,7 @@ import os
from ConfigParser import ConfigParser, NoSectionError, NoOptionError, \
RawConfigParser
from webob.exc import HTTPBadRequest, HTTPLengthRequired, \
from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \
HTTPRequestEntityTooLarge
constraints_conf = ConfigParser()

10
swift/common/db_replicator.py
View File

@ -26,18 +26,18 @@ import re
from eventlet import GreenPool, sleep, Timeout
from eventlet.green import subprocess
import simplejson
from webob import Response
from webob.exc import HTTPNotFound, HTTPNoContent, HTTPAccepted, \
HTTPInsufficientStorage, HTTPBadRequest
import swift.common.db
from swift.common.utils import get_logger, whataremyips, storage_directory, \
renamer, mkdirs, lock_parent_directory, TRUE_VALUES, unlink_older_than, \
dump_recon_cache, rsync_ip
from swift.common import ring
from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE
from swift.common.bufferedhttp import BufferedHTTPConnection
from swift.common.exceptions import DriveNotMounted, ConnectionTimeout
from swift.common.daemon import Daemon
from swift.common.swob import Response, HTTPNotFound, HTTPNoContent, \
HTTPAccepted, HTTPInsufficientStorage, HTTPBadRequest
DEBUG_TIMINGS_THRESHOLD = 10
@ -324,11 +324,11 @@ class Replicator(Daemon):
info['delete_timestamp'], info['metadata'])
if not response:
return False
elif response.status == HTTPNotFound.code: # completely missing, rsync
elif response.status == HTTP_NOT_FOUND: # completely missing, rsync
self.stats['rsync'] += 1
self.logger.increment('rsyncs')
return self._rsync_db(broker, node, http, info['id'])
elif response.status == HTTPInsufficientStorage.code:
elif response.status == HTTP_INSUFFICIENT_STORAGE:
raise DriveNotMounted()
elif response.status >= 200 and response.status < 300:
rinfo = simplejson.loads(response.data)

34
swift/common/http.py
View File

@ -13,40 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from webob.exc import HTTPClientError,\
HTTPInsufficientStorage as BaseHTTPInsufficientStorage
class HTTPClientDisconnect(HTTPClientError):
"""
subclass of :class:`~HTTPClientError`
This code is introduced to log the case when the connection is closed by
client while HTTP server is processing its request
code: 499, title: Client Disconnect
"""
code = 499
title = 'Client Disconnect'
explanation = (
'This code is introduced to log the case when the connection '
'is closed by client while HTTP server is processing its request')
class HTTPInsufficientStorage(BaseHTTPInsufficientStorage):
"""
subclass of :class:`~HTTPInsufficientStorage`
The server is unable to store the representation needed to
complete the request.
code: 507, title: Insufficient Storage
"""
def __init__(self, drive=None, *args, **kwargs):
if drive:
self.explanation = ('%s is not mounted' % drive)
super(HTTPInsufficientStorage, self).__init__(*args, **kwargs)
def is_informational(status):
"""

3
swift/common/internal_client.py
View File

@ -19,12 +19,11 @@ from paste.deploy import loadapp
import struct
from sys import exc_info
from urllib import quote
from webob import Request
import zlib
from zlib import compressobj
from swift.common.http import HTTP_NOT_FOUND
from swift.common.swob import Request
class UnexpectedResponse(Exception):

3
swift/common/middleware/catch_errors.py
View File

@ -14,10 +14,9 @@
# limitations under the License.
from eventlet import Timeout
from webob import Request
from webob.exc import HTTPServerError
import uuid
from swift.common.swob import Request, HTTPServerError
from swift.common.utils import get_logger

3
swift/common/middleware/cname_lookup.py
View File

@ -27,8 +27,6 @@ maximum lookup depth. If a match is found, the environment's Host header is
rewritten and the request is passed further down the WSGI chain.
"""
from webob import Request
from webob.exc import HTTPBadRequest
try:
import dns.resolver
from dns.exception import DNSException
@ -39,6 +37,7 @@ except ImportError:
else: # executed if the try block finishes with no errors
MODULE_DEPENDENCY_MET = True
from swift.common.swob import Request, HTTPBadRequest
from swift.common.utils import cache_from_env, get_logger

3
swift/common/middleware/domain_remap.py
View File

@ -49,8 +49,7 @@ advised. With container sync, you should use the true storage end points as
sync destinations.
"""
from webob import Request
from webob.exc import HTTPBadRequest
from swift.common.swob import Request, HTTPBadRequest
class DomainRemapMiddleware(object):

2
swift/common/middleware/healthcheck.py
View File

@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from webob import Request, Response
from swift.common.swob import Request, Response
class HealthCheckMiddleware(object):

11
swift/common/middleware/keystoneauth.py
View File

@ -14,10 +14,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import webob
from swift.common import utils as swift_utils
from swift.common.middleware import acl as swift_acl
from swift.common.swob import HTTPNotFound, HTTPForbidden, HTTPUnauthorized
class KeystoneAuth(object):
@ -153,7 +152,7 @@ class KeystoneAuth(object):
part = swift_utils.split_path(req.path, 1, 4, True)
version, account, container, obj = part
except ValueError:
return webob.exc.HTTPNotFound(request=req)
return HTTPNotFound(request=req)
user_roles = env_identity.get('roles', [])
@ -226,7 +225,7 @@ class KeystoneAuth(object):
part = swift_utils.split_path(req.path, 1, 4, True)
version, account, container, obj = part
except ValueError:
return webob.exc.HTTPNotFound(request=req)
return HTTPNotFound(request=req)
is_authoritative_authz = (account and
account.startswith(self.reseller_prefix))
@ -274,9 +273,9 @@ class KeystoneAuth(object):
depending on whether the REMOTE_USER is set or not.
"""
if req.remote_user:
return webob.exc.HTTPForbidden(request=req)
return HTTPForbidden(request=req)
else:
return webob.exc.HTTPUnauthorized(request=req)
return HTTPUnauthorized(request=req)
def filter_factory(global_conf, **local_conf):

5
swift/common/middleware/name_check.py
View File

@ -38,10 +38,11 @@ The filter returns HTTPBadRequest if path is invalid.
import re
from swift.common.utils import get_logger
from webob import Request
from webob.exc import HTTPBadRequest
from urllib2 import unquote
from swift.common.swob import Request, HTTPBadRequest
FORBIDDEN_CHARS = "\'\"`<>"
MAX_LENGTH = 255
FORBIDDEN_REGEXP = "/\./|/\.\./|/\.$|/\.\.$"

3
swift/common/middleware/proxy_logging.py
View File

@ -40,8 +40,7 @@ be separated with a simple .split()
import time
from urllib import quote, unquote
from webob import Request
from swift.common.swob import Request
from swift.common.utils import (get_logger, get_remote_client,
get_valid_utf8_str, TRUE_VALUES)

4
swift/common/middleware/ratelimit.py
View File

@ -13,11 +13,11 @@
# limitations under the License.
import time
import eventlet
from webob import Request, Response
from swift.common.utils import split_path, cache_from_env, get_logger
from swift.proxy.controllers.base import get_container_memcache_key
from swift.common.memcached import MemcacheConnectionError
from swift.common.swob import Request, Response
class MaxSleepTimeHitError(Exception):
@ -205,7 +205,7 @@ class RateLimitMiddleware(object):
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in webob.Request object and passes it down.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable

2
swift/common/middleware/recon.py
View File

@ -16,7 +16,7 @@
import errno
import os
from webob import Request, Response
from swift.common.swob import Request, Response
from swift.common.utils import split_path, get_logger, TRUE_VALUES
from swift.common.constraints import check_mount
from resource import getpagesize

3
swift/common/middleware/staticweb.py
View File

@ -118,14 +118,13 @@ import cgi
import time
from urllib import unquote, quote as urllib_quote
from webob import Response
from webob.exc import HTTPMovedPermanently, HTTPNotFound
from swift.common.utils import cache_from_env, get_logger, human_readable, \
split_path, TRUE_VALUES
from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request, \
WSGIContext
from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND
from swift.common.swob import Response, HTTPMovedPermanently, HTTPNotFound
def quote(value, safe='/'):

14
swift/common/middleware/tempauth.py
View File

@ -22,8 +22,8 @@ import hmac
import base64
from eventlet import Timeout
from webob import Response, Request
from webob.exc import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \
from swift.common.swob import Response, Request
from swift.common.swob import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \
HTTPUnauthorized
from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
@ -285,7 +285,7 @@ class TempAuth(object):
"""
WSGI entry point for auth requests (ones that match the
self.auth_prefix).
Wraps env in webob.Request object and passes it down.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
@ -321,9 +321,9 @@ class TempAuth(object):
def handle_request(self, req):
"""
Entry point for auth requests (ones that match the self.auth_prefix).
Should return a WSGI-style callable (such as webob.Response).
Should return a WSGI-style callable (such as swob.Response).
:param req: webob.Request object
:param req: swob.Request object
"""
req.start_time = time()
handler = None
@ -363,8 +363,8 @@ class TempAuth(object):
X-Storage-Token set to the token to use with Swift and X-Storage-URL
set to the URL to the default Swift cluster to use.
:param req: The webob.Request to process.
:returns: webob.Response, 2xx on success with data set as explained
:param req: The swob.Request to process.
:returns: swob.Response, 2xx on success with data set as explained
above.
"""
# Validate the request info

840
swift/common/swob.py Normal file
View File

@ -0,0 +1,840 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Implementation of WSGI Request and Response objects.
This library has a very similar API to Webob. It wraps WSGI request
environments and response values into objects that are more friendly to
interact with.
"""
from cStringIO import StringIO
import UserDict
import time
from functools import partial
from datetime import datetime, date, timedelta, tzinfo
from email.utils import parsedate
import urlparse
import urllib2
import re
from swift.common.utils import reiterate
RESPONSE_REASONS = {
100: ('Continue', ''),
200: ('OK', ''),
201: ('Created', ''),
202: ('Accepted', 'The request is accepted for processing.'),
204: ('No Content', ''),
206: ('Partial Content', ''),
301: ('Moved Permanently', 'The resource has moved permanently.'),
302: ('Found', ''),
304: ('Not Modified', ''),
307: ('Temporary Redirect', 'The resource has moved temporarily.'),
400: ('Bad Request', 'The server could not comply with the request since '
'it is either malformed or otherwise incorrect.'),
401: ('Unauthorized', 'This server could not verify that you are '
'authorized to access the document you requested.'),
402: ('Payment Required', 'Access was denied for financial reasons.'),
403: ('Forbidden', 'Access was denied to this resource.'),
404: ('Not Found', 'The resource could not be found.'),
405: ('Method Not Allowed', 'The method is not allowed for this '
'resource.'),
406: ('Not Acceptable', 'The resource is not available in a format '
'acceptable to your browser.'),
408: ('Request Timeout', 'The server has waited too long for the request '
'to be sent by the client.'),
409: ('Conflict', 'There was a conflict when trying to complete '
'your request.'),
410: ('Gone', 'This resource is no longer available.'),
411: ('Length Required', 'Content-Length header required.'),
412: ('Precondition Failed', 'A precondition for this request was not '
'met.'),
413: ('Request Entity Too Large', 'The body of your request was too '
'large for this server.'),
414: ('Request URI Too Long', 'The request URI was too long for this '
'server.'),
415: ('Unsupported Media Type', 'The request media type is not '
'supported by this server.'),
416: ('Request Range Not Satisfiable', 'The Range requested is not '
'available.'),
417: ('Expectation Failed', 'Expectation failed.'),
422: ('Unprocessable Entity', 'Unable to process the contained '
'instructions'),
499: ('Client Disconnect', 'The client was disconnected during request.'),
500: ('Internal Error', 'The server has either erred or is incapable of '
'performing the requested operation.'),
501: ('Not Implemented', 'The requested method is not implemented by '
'this server.'),
502: ('Bad Gateway', 'Bad gateway.'),
503: ('Service Unavailable', 'The server is currently unavailable. '
'Please try again at a later time.'),
504: ('Gateway Timeout', 'A timeout has occurred speaking to a '
'backend server.'),
507: ('Insufficient Storage', 'There was not enough space to save the '
'resource.'),
}
class _UTC(tzinfo):
"""
A tzinfo class for datetime objects that returns a 0 timedelta (UTC time)
"""
def dst(self, dt):
return timedelta(0)
utcoffset = dst
def tzname(self, dt):
return 'UTC'
UTC = _UTC()
def _datetime_property(header):
"""
Set and retrieve the datetime value of self.headers[header]
(Used by both request and response)
The header is parsed on retrieval and a datetime object is returned.
The header can be set using a datetime, numeric value, or str.
If a value of None is given, the header is deleted.
:param header: name of the header, e.g. "Content-Length"
"""
def getter(self):
value = self.headers.get(header, None)
if value is not None:
try:
parts = parsedate(self.headers[header])[:7]
date = datetime(*(parts + (UTC,)))
except Exception:
return None
if date.year < 1970:
raise ValueError('Somehow an invalid year')
return date
def setter(self, value):
if isinstance(value, (float, int, long)):
self.headers[header] = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(value))
elif isinstance(value, datetime):
self.headers[header] = value.strftime("%a, %d %b %Y %H:%M:%S GMT")
else:
self.headers[header] = value
return property(getter, setter,
doc=("Retrieve and set the %s header as a datetime, "
"set it with a datetime, int, or str") % header)
def _header_property(header):
"""
Set and retrieve the value of self.headers[header]
(Used by both request and response)
If a value of None is given, the header is deleted.
:param header: name of the header, e.g. "Content-Length"
"""
def getter(self):
return self.headers.get(header, None)
def setter(self, value):
self.headers[header] = value
return property(getter, setter,
doc="Retrieve and set the %s header" % header)
def _header_int_property(header):
"""
Set and retrieve the value of self.headers[header]
(Used by both request and response)
On retrieval, it converts values to integers.
If a value of None is given, the header is deleted.
:param header: name of the header, e.g. "Content-Length"
"""
def getter(self):
val = self.headers.get(header, None)
if val is not None:
val = int(val)
return val
def setter(self, value):
self.headers[header] = value
return property(getter, setter,
doc="Retrieve and set the %s header as an int" % header)
class HeaderEnvironProxy(UserDict.DictMixin):
"""
A dict-like object that proxies requests to a wsgi environ,
rewriting header keys to environ keys.
For example, headers['Content-Range'] sets and gets the value of
headers.environ['HTTP_CONTENT_RANGE']
"""
def __init__(self, environ):
self.environ = environ
def _normalize(self, key):
key = 'HTTP_' + key.replace('-', '_').upper()
if key == 'HTTP_CONTENT_LENGTH':
return 'CONTENT_LENGTH'
if key == 'HTTP_CONTENT_TYPE':
return 'CONTENT_TYPE'
return key
def __getitem__(self, key):
return self.environ[self._normalize(key)]
def __setitem__(self, key, value):
if value is None:
self.environ.pop(self._normalize(key), None)
elif isinstance(value, unicode):
self.environ[self._normalize(key)] = value.encode('utf-8')
else:
self.environ[self._normalize(key)] = str(value)
def __contains__(self, key):
return self._normalize(key) in self.environ
def __delitem__(self, key):
del self.environ[self._normalize(key)]
def keys(self):
keys = [key[5:].replace('_', '-').title()
for key in self.environ.iterkeys() if key.startswith('HTTP_')]
if 'CONTENT_LENGTH' in self.environ:
keys.append('Content-Length')
if 'CONTENT_TYPE' in self.environ:
keys.append('Content-Type')
return keys
class HeaderKeyDict(dict):
"""
A dict that lower-cases all keys on the way in, so as to be
case-insensitive.
"""
def __init__(self, *args, **kwargs):
for arg in args:
self.update(arg)
self.update(kwargs)
def update(self, other):
if hasattr(other, 'keys'):
for key in other.keys():
self[key.lower()] = other[key]
else:
for key, value in other:
self[key.lower()] = value
def __getitem__(self, key):
return dict.get(self, key.lower())
def __setitem__(self, key, value):
if value is None:
self.pop(key.lower(), None)
elif isinstance(value, unicode):
return dict.__setitem__(self, key.lower(), value.encode('utf-8'))
else:
return dict.__setitem__(self, key.lower(), str(value))
def __contains__(self, key):
return dict.__contains__(self, key.lower())
def __delitem__(self, key):
return dict.__delitem__(self, key.lower())
def get(self, key, default=None):
return dict.get(self, key.lower(), default)
def _resp_status_property():
"""
Set and retrieve the value of Response.status
On retrieval, it concatenates status_int and title.
When set to a str, it splits status_int and title apart.
When set to an integer, retrieves the correct title for that
response code from the RESPONSE_REASONS dict.
:param header: name of the header, e.g. "Content-Length"
"""
def getter(self):
return '%s %s' % (self.status_int, self.title)
def setter(self, value):
if isinstance(value, (int, long)):
self.status_int = value
self.explanation = self.title = RESPONSE_REASONS[value][0]
else:
if isinstance(value, unicode):
value = value.encode('utf-8')
self.status_int = int(value.split(' ', 1)[0])
self.explanation = self.title = value.split(' ', 1)[1]
return property(getter, setter,
doc="Retrieve and set the Response status, e.g. '200 OK'")
def _resp_body_property():
"""
Set and retrieve the value of Response.body
If necessary, it will consume Response.app_iter to create a body.
On assignment, encodes unicode values to utf-8, and sets the content-length
to the length of the str.
"""
def getter(self):
if not self._body:
self._body = ''.join(self._app_iter)
self._app_iter = None
return self._body
def setter(self, value):
if isinstance(value, unicode):
value = value.encode('utf-8')
if isinstance(value, str):
self.content_length = len(value)
self._app_iter = None
self._body = value
return property(getter, setter,
doc="Retrieve and set the Response body str")
def _resp_etag_property():
"""
Set and retrieve Response.etag
This may be broken for etag use cases other than Swift's.
Quotes strings when assigned and unquotes when read, for compatibility
with webob.
"""
def getter(self):
etag = self.headers.get('etag', None)
if etag:
etag = etag.replace('"', '')
return etag
def setter(self, value):
if value is None:
self.headers['etag'] = None
else:
self.headers['etag'] = '"%s"' % value
return property(getter, setter,
doc="Retrieve and set the response Etag header")
def _resp_content_type_property():
"""
Set and retrieve Response.content_type
Strips off any charset when retrieved -- that is accessible
via Response.charset.
"""
def getter(self):
if 'content-type' in self.headers:
return self.headers.get('content-type').split(';')[0]
def setter(self, value):
self.headers['content-type'] = value
return property(getter, setter,
doc="Retrieve and set the response Content-Type header")
def _resp_charset_property():
"""
Set and retrieve Response.charset
On retrieval, separates the charset from the content-type.
On assignment, removes any existing charset from the content-type and
appends the new one.
"""
def getter(self):
if '; charset=' in self.headers['content-type']:
return self.headers['content-type'].split('; charset=')[1]
def setter(self, value):
if 'content-type' in self.headers:
self.headers['content-type'] = self.headers['content-type'].split(
';')[0]
if value:
self.headers['content-type'] += '; charset=' + value
return property(getter, setter,
doc="Retrieve and set the response charset")
def _resp_app_iter_property():
"""
Set and retrieve Response.app_iter
Mostly a pass-through to Response._app_iter, it's a property so it can zero
out an exsisting content-length on assignment.
"""
def getter(self):
return self._app_iter
def setter(self, value):
if isinstance(value, (list, tuple)):
self.content_length = sum(map(len, value))
elif value is not None:
self.content_length = None
self._body = None
self._app_iter = value
return property(getter, setter,
doc="Retrieve and set the response app_iter")
def _req_fancy_property(cls, header, even_if_nonexistent=False):
"""
Set and retrieve "fancy" properties.
On retrieval, these properties return a class that takes the value of the
header as the only argument to their constructor.
For assignment, those classes should implement a __str__ that converts them
back to their header values.
:param header: name of the header, e.g. "Accept"
:param even_if_nonexistent: Return a value even if the header does not
exist. Classes using this should be prepared to accept None as a
parameter.
"""
def getter(self):
try:
if header in self.headers or even_if_nonexistent:
return cls(self.headers.get(header))
except ValueError:
return None
def setter(self, value):
self.headers[header] = value
return property(getter, setter, doc=("Retrieve and set the %s "
"property in the WSGI environ, as a %s object") %
(header, cls.__name__))
class Range(object):
"""
Wraps a Request's Range header as a friendly object.
After initialization, "range.ranges" is populated with a list
of (start, end) tuples denoting the requested ranges.
:param headerval: value of the header as a str
"""
def __init__(self, headerval):
headerval = headerval.replace(' ', '')
if not headerval.lower().startswith('bytes='):
raise ValueError('Invalid Range header: %s' % headerval)
self.ranges = []
for rng in headerval[6:].split(','):
start, end = rng.split('-', 1)
if start:
start = int(start)
else:
start = None
if end:
end = int(end)
else:
end = None
self.ranges.append((start, end))
def __str__(self):
string = 'bytes='
for start, end in self.ranges:
if start is not None:
string += str(start)
string += '-'
if end is not None:
string += str(end)
string += ','
return string.rstrip(',')
def range_for_length(self, length):
"""
range_for_length is used to determine the correct range of bytes to
serve from a body, given body length argument and the Range's ranges.
A limitation of this method is that it can't handle multiple ranges,
for compatibility with webob. This should be fairly easy to extend.
:param length: length of the response body
"""
if length is None or not self.ranges or len(self.ranges) != 1:
return None
begin, end = self.ranges[0]
if begin is None:
if end == 0:
return (0, length)
if end > length:
return None
return (length - end, length)
if end is None:
if begin == 0:
return (0, length)
return (begin, length)
if begin > length:
return None
return (begin, min(end + 1, length))
class Match(object):
"""
Wraps a Request's If-None-Match header as a friendly object.
:param headerval: value of the header as a str
"""
def __init__(self, headerval):
self.tags = set()
for tag in headerval.split(', '):
if tag.startswith('"') and tag.endswith('"'):
self.tags.add(tag[1:-1])
else:
self.tags.add(tag)
def __contains__(self, val):
return '*' in self.tags or val in self.tags
class Accept(object):
"""
Wraps a Request's Accept header as a friendly object.
:param headerval: value of the header as a str
"""
def __init__(self, headerval):
self.headerval = headerval
def _get_types(self):
headerval = self.headerval or '*/*'
level = 1
types = []
for typ in headerval.split(','):
quality = 1.0
if '; q=' in typ:
typ, quality = typ.split('; q=')
elif ';q=' in typ:
typ, quality = typ.split(';q=')
quality = float(quality)
if typ.startswith('*/'):
quality -= 0.01
elif typ.endswith('/*'):
quality -= 0.01
elif '*' in typ:
raise AssertionError('bad accept header')
pattern = '[a-zA-Z0-9-]+'.join([re.escape(x) for x in
typ.strip().split('*')])
types.append((quality, re.compile(pattern), typ))
types.sort(reverse=True, key=lambda t: t[0])
return types
def best_match(self, options, default_match='text/plain'):
for quality, pattern, typ in self._get_types():
for option in options:
if pattern.match(option):
return option
return default_match
def __repr__(self):
return self.headerval
def _req_environ_property(environ_field):
"""
Set and retrieve value of the environ_field entry in self.environ.
(Used by both request and response)
"""
def getter(self):
return self.environ.get(environ_field, None)
def setter(self, value):
self.environ[environ_field] = value
return property(getter, setter, doc=("Get and set the %s property "
"in the WSGI environment") % environ_field)
def _req_body_property():
"""
Set and retrieve the Request.body parameter. It consumes wsgi.input and
returns the results. On assignment, uses a StringIO to create a new
wsgi.input.
"""
def getter(self):
body = self.environ['wsgi.input'].read()
self.environ['wsgi.input'] = StringIO(body)
return body
def setter(self, value):
self.environ['wsgi.input'] = StringIO(value)
self.environ['CONTENT_LENGTH'] = str(len(value))
return property(getter, setter, doc="Get and set the request body str")
class Request(object):
"""
WSGI Request object.
"""
range = _req_fancy_property(Range, 'range')
if_none_match = _req_fancy_property(Match, 'if-none-match')
accept = _req_fancy_property(Accept, 'http-accept', True)
method = _req_environ_property('REQUEST_METHOD')
referrer = referer = _req_environ_property('HTTP_REFERER')
script_name = _req_environ_property('SCRIPT_NAME')
path_info = _req_environ_property('PATH_INFO')
host = _req_environ_property('HTTP_HOST')
remote_addr = _req_environ_property('REMOTE_ADDR')
remote_user = _req_environ_property('REMOTE_USER')
user_agent = _req_environ_property('HTTP_USER_AGENT')
query_string = _req_environ_property('QUERY_STRING')
if_match = _req_environ_property('HTTP_IF_MATCH')
body_file = _req_environ_property('wsgi.input')
content_length = _header_int_property('content-length')
if_modified_since = _datetime_property('if-modified-since')
if_unmodified_since = _datetime_property('if-unmodified-since')
body = _req_body_property()
charset = None
_params_cache = None
acl = _req_environ_property('swob.ACL')
def __init__(self, environ):
self.environ = environ
self.headers = HeaderEnvironProxy(self.environ)
@classmethod
def blank(cls, path, environ=None, headers=None, body=None):
"""
Create a new request object with the given parameters, and an
environment otherwise filled in with non-surprising default values.
"""
headers = headers or {}
environ = environ or {}
if '?' in path:
path_info, query_string = path.split('?')
else:
path_info = path
query_string = ''
env = {
'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'QUERY_STRING': query_string,
'PATH_INFO': path_info,
'SERVER_NAME': 'localhost',
'SERVER_PORT': '80',
'HTTP_HOST': 'localhost:80',
'SERVER_PROTOCOL': 'HTTP/1.0',
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http',
'wsgi.input': StringIO(body or ''),
'wsgi.errors': StringIO(''),
'wsgi.multithread': False,
'wsgi.multiprocess': False
}
env.update(PATH_INFO=path_info)
env.update(environ)
if body is not None:
env.update(CONTENT_LENGTH=str(len(body)))
req = Request(env)
for key, val in headers.iteritems():
req.headers[key] = val
return req
@property
def params(self):
"Provides QUERY_STRING parameters as a dictionary"
if self._params_cache is None:
if 'QUERY_STRING' in self.environ:
self._params_cache = dict(
urlparse.parse_qsl(self.environ['QUERY_STRING'], True))
else:
self._params_cache = {}
return self._params_cache
str_params = params
@property
def path(self):
"Provides the full path of the request, excluding the QUERY_STRING"
return urllib2.quote(self.environ.get('SCRIPT_NAME', '') +
self.environ['PATH_INFO'].split('?')[0])
def path_info_pop(self):
"""
Takes one path portion (delineated by slashes) from the
path_info, and appends it to the script_name. Returns
the path segment.
"""
path_info = self.path_info
try:
slash_loc = path_info.index('/', 1)
except ValueError:
return None
self.script_name += path_info[:slash_loc]
self.path_info = path_info[slash_loc:]
return path_info[1:slash_loc]
def copy_get(self):
"""
Makes a copy of the request, converting it to a GET.
"""
env = self.environ.copy()
env.update({
'REQUEST_METHOD': 'GET',
'CONTENT_LENGTH': '0',
'wsgi.input': StringIO(''),
})
return Request(env)
def call_application(self, application):
"""
Calls the application with this request's environment. Returns the
status, headers, and app_iter for the response as a tuple.
:param application: the WSGI application to call
"""
output = []
captured = []
def start_response(status, headers, exc_info=None):
captured[:] = [status, headers, exc_info]
return output.append
app_iter = application(self.environ, start_response)
if not app_iter:
app_iter = output
if not captured:
app_iter = reiterate(app_iter)
return (captured[0], captured[1], app_iter)
def get_response(self, application):
"""
Calls the application with this request's environment. Returns a
Response object that wraps up the application's result.
:param application: the WSGI application to call
"""
status, headers, app_iter = self.call_application(application)
return Response(status=status, headers=dict(headers),
app_iter=app_iter, request=self)
class Response(object):
"""
WSGI Response object.
"""
content_length = _header_int_property('content-length')
content_type = _resp_content_type_property()
content_range = _header_property('content-range')
etag = _resp_etag_property()
status = _resp_status_property()
body = _resp_body_property()
last_modified = _datetime_property('last-modified')
location = _header_property('location')
accept_ranges = _header_property('accept-ranges')
charset = _resp_charset_property()
app_iter = _resp_app_iter_property()
def __init__(self, body=None, status=200, headers={}, app_iter=None,
request=None, conditional_response=False, **kw):
self.headers = HeaderKeyDict()
self.conditional_response = conditional_response
self.request = request
self.body = body
self.app_iter = app_iter
self.status = status
if request:
self.environ = request.environ
if request.range and self.status == 200:
self.status = 206
else:
self.environ = {}
self.headers.update(headers)
for key, value in kw.iteritems():
setattr(self, key, value)
def _response_iter(self, app_iter, body):
if self.request and self.request.method == 'HEAD':
return ['']
if self.conditional_response and self.request and \
self.request.range and not self.content_range:
args = self.request.range.range_for_length(self.content_length)
if not args:
self.status = 416
else:
start, end = args
self.status = 206
self.content_range = self.request.range
self.content_length = (end - start)
if app_iter and hasattr(app_iter, 'app_iter_range'):
return app_iter.app_iter_range(start, end)
elif app_iter:
# this could be improved, but we don't actually use it
return [''.join(app_iter)[start:end]]
elif body:
return [body[start:end]]
if app_iter:
return app_iter
if body:
return [body]
if self.status_int in RESPONSE_REASONS:
title, exp = RESPONSE_REASONS[self.status_int]
if exp:
body = '<html><h1>%s</h1><p>%s</p></html>' % (title, exp)
self.content_length = len(body)
self.content_type = 'text/html'
return [body]
return ['']
def __call__(self, env, start_response):
self.environ = env
app_iter = self._response_iter(self.app_iter, self._body)
if 'location' in self.headers and self.location.startswith('/'):
self.location = self.environ['wsgi.url_scheme'] + '://' \
+ self.environ['SERVER_NAME'] + self.location
start_response(self.status, self.headers.items())
return app_iter
class StatusMap(object):
"""
A dict-like object that returns Response subclasses/factory functions
where the given key is the status code.
"""
def __getitem__(self, key):
return partial(Response, status=key)
status_map = StatusMap()
HTTPAccepted = status_map[202]
HTTPCreated = status_map[201]
HTTPNoContent = status_map[204]
HTTPMovedPermanently = status_map[301]
HTTPNotModified = status_map[304]
HTTPBadRequest = status_map[400]
HTTPUnauthorized = status_map[401]
HTTPForbidden = status_map[403]
HTTPMethodNotAllowed = status_map[405]
HTTPNotFound = status_map[404]
HTTPRequestTimeout = status_map[408]
HTTPConflict = status_map[409]
HTTPLengthRequired = status_map[411]
HTTPPreconditionFailed = status_map[412]
HTTPRequestEntityTooLarge = status_map[413]
HTTPUnprocessableEntity = status_map[422]
HTTPClientDisconnect = status_map[499]
HTTPServerError = status_map[500]
HTTPInternalServerError = status_map[500]
HTTPServiceUnavailable = status_map[503]
HTTPInsufficientStorage = status_map[507]

25
swift/common/utils.py
View File

@ -40,6 +40,8 @@ import cPickle as pickle
import glob
from urlparse import urlparse as stdlib_urlparse, ParseResult
import socket
import itertools
import types
import eventlet
from eventlet import GreenPool, sleep, Timeout
@ -114,7 +116,7 @@ def get_param(req, name, default=None):
Get parameters from an HTTP request ensuring proper handling UTF-8
encoding.
:param req: Webob request object
:param req: request object
:param name: parameter name
:param default: result to return if the parameter is not found
:returns: HTTP request parameter value
@ -1440,3 +1442,24 @@ def list_from_csv(comma_separated_str):
if comma_separated_str:
return [v.strip() for v in comma_separated_str.split(',') if v.strip()]
return []
def reiterate(iterable):
"""
Consume the first item from an iterator, then re-chain it to the rest of
the iterator. This is useful when you want to make sure the prologue to
downstream generators have been executed before continuing.
:param iterable: an iterable object
"""
if isinstance(iterable, (list, tuple)):
return iterable
else:
iterator = iter(iterable)
try:
chunk = ''
while not chunk:
chunk = next(iterable)
return itertools.chain([chunk], iterable)
except StopIteration:
return []

6
swift/common/wsgi.py
View File

@ -27,9 +27,9 @@ import eventlet
from eventlet import greenio, GreenPool, sleep, wsgi, listen
from paste.deploy import loadapp, appconfig
from eventlet.green import socket, ssl
from webob import Request
from urllib import unquote
from swift.common.swob import Request
from swift.common.utils import capture_stdio, disable_fallocate, \
drop_privileges, get_logger, NullLogger, TRUE_VALUES, \
validate_configuration
@ -265,7 +265,7 @@ class WSGIContext(object):
def make_pre_authed_request(env, method=None, path=None, body=None,
headers=None, agent='Swift'):
"""
Makes a new webob.Request based on the current env but with the
Makes a new swob.Request based on the current env but with the
parameters specified. Note that this request will be preauthorized.
:param env: The WSGI environment to base the new request on.
@ -285,7 +285,7 @@ def make_pre_authed_request(env, method=None, path=None, body=None,
'%(orig)s StaticWeb'. You also set agent to None to
use the original env's HTTP_USER_AGENT or '' to
have no HTTP_USER_AGENT.
:returns: Fresh webob.Request object.
:returns: Fresh swob.Request object.
"""
query_string = None
if path and '?' in path:

11
swift/container/server.py
View File

@ -23,10 +23,6 @@ from xml.sax import saxutils
from datetime import datetime
from eventlet import Timeout
from webob import Request, Response
from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
HTTPCreated, HTTPInternalServerError, HTTPNoContent, \
HTTPNotFound, HTTPPreconditionFailed, HTTPMethodNotAllowed
import swift.common.db
from swift.common.db import ContainerBroker
@ -38,7 +34,10 @@ from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
from swift.common.bufferedhttp import http_connect
from swift.common.exceptions import ConnectionTimeout
from swift.common.db_replicator import ReplicatorRpc
from swift.common.http import HTTP_NOT_FOUND, is_success, \
from swift.common.http import HTTP_NOT_FOUND, is_success
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
HTTPCreated, HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
HTTPPreconditionFailed, HTTPMethodNotAllowed, Request, Response, \
HTTPInsufficientStorage
DATADIR = 'containers'
@ -90,7 +89,7 @@ class ContainerController(object):
"""
Update the account server with latest container info.
:param req: webob.Request object
:param req: swob.Request object
:param account: account name
:param container: container name
:param borker: container DB broker object

13
swift/obj/server.py
View File

@ -27,11 +27,6 @@ from tempfile import mkstemp
from urllib import unquote
from contextlib import contextmanager
from webob import Request, Response, UTC
from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
HTTPNotModified, HTTPPreconditionFailed, \
HTTPRequestTimeout, HTTPUnprocessableEntity, HTTPMethodNotAllowed
from xattr import getxattr, setxattr
from eventlet import sleep, Timeout, tpool
@ -46,8 +41,12 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileError, \
DiskFileNotExist
from swift.obj.replicator import tpool_reraise, invalidate_hash, \
quarantine_renamer, get_hashes
from swift.common.http import is_success, HTTPInsufficientStorage, \
HTTPClientDisconnect
from swift.common.http import is_success
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, UTC, \
HTTPInsufficientStorage
DATADIR = 'objects'

4
swift/proxy/controllers/account.py
View File

@ -28,13 +28,11 @@ import time
from urllib import unquote
from random import shuffle
from webob.exc import HTTPBadRequest, HTTPMethodNotAllowed
from webob import Request
from swift.common.utils import normalize_timestamp, public
from swift.common.constraints import check_metadata, MAX_ACCOUNT_NAME_LENGTH
from swift.common.http import is_success, HTTP_NOT_FOUND
from swift.proxy.controllers.base import Controller
from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed, Request
class AccountController(Controller):

15
swift/proxy/controllers/base.py
View File

@ -30,8 +30,6 @@ import functools
from eventlet import spawn_n, GreenPile, Timeout
from eventlet.queue import Queue, Empty, Full
from eventlet.timeout import Timeout
from webob.exc import status_map
from webob import Request, Response
from swift.common.utils import normalize_timestamp, TRUE_VALUES, public
from swift.common.bufferedhttp import http_connect
@ -41,13 +39,14 @@ from swift.common.http import is_informational, is_success, is_redirection, \
is_server_error, HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_MULTIPLE_CHOICES, \
HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVICE_UNAVAILABLE, \
HTTP_INSUFFICIENT_STORAGE
from swift.common.swob import Request, Response, status_map
def update_headers(response, headers):
"""
Helper function to update headers in the response.
:param response: webob.Response object
:param response: swob.Response object
:param headers: dictionary headers
"""
if hasattr(headers, 'items'):
@ -406,7 +405,7 @@ class Controller(object):
:param headers: a list of dicts, where each dict represents one
backend request that should be made.
:returns: a webob Response object
:returns: a swob.Response object
"""
start_nodes = ring.get_part_nodes(part)
nodes = self.iter_nodes(part, start_nodes, ring)
@ -427,13 +426,13 @@ class Controller(object):
Given a list of responses from several servers, choose the best to
return to the API.
:param req: webob.Request object
:param req: swob.Request object
:param statuses: list of statuses returned
:param reasons: list of reasons for each status
:param bodies: bodies of each response
:param server_type: type of server the responses came from
:param etag: etag
:returns: webob.Response object with the correct status, body, etc. set
:returns: swob.Response object with the correct status, body, etc. set
"""
resp = Response(request=req)
if len(statuses):
@ -562,13 +561,13 @@ class Controller(object):
"""
Base handler for HTTP GET or HEAD requests.
:param req: webob.Request object
:param req: swob.Request object
:param server_type: server type
:param partition: partition
:param nodes: nodes
:param path: path for the request
:param attempts: number of attempts to try
:returns: webob.Response object
:returns: swob.Response object
"""
statuses = []
reasons = []

4
swift/proxy/controllers/container.py
View File

@ -28,13 +28,13 @@ import time
from urllib import unquote
from random import shuffle
from webob.exc import HTTPBadRequest, HTTPForbidden, HTTPNotFound
from swift.common.utils import normalize_timestamp, public
from swift.common.constraints import check_metadata, MAX_CONTAINER_NAME_LENGTH
from swift.common.http import HTTP_ACCEPTED
from swift.proxy.controllers.base import Controller, delay_denial, \
get_container_memcache_key
from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
HTTPNotFound
class ContainerController(Controller):

18
swift/proxy/controllers/obj.py
View File

@ -39,10 +39,6 @@ from random import shuffle
from eventlet import sleep, GreenPile, Timeout
from eventlet.queue import Queue
from eventlet.timeout import Timeout
from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \
HTTPServerError, HTTPServiceUnavailable
from webob import Request, Response
from swift.common.utils import ContextPool, normalize_timestamp, TRUE_VALUES, \
public
@ -55,8 +51,12 @@ from swift.common.exceptions import ChunkReadTimeout, \
from swift.common.http import is_success, is_client_error, HTTP_CONTINUE, \
HTTP_CREATED, HTTP_MULTIPLE_CHOICES, HTTP_NOT_FOUND, \
HTTP_INTERNAL_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE, \
HTTP_INSUFFICIENT_STORAGE, HTTPClientDisconnect
HTTP_INSUFFICIENT_STORAGE
from swift.proxy.controllers.base import Controller, delay_denial
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \
HTTPServerError, HTTPServiceUnavailable, Request, Response, \
HTTPClientDisconnect
class SegmentedIterable(object):
@ -72,7 +72,7 @@ class SegmentedIterable(object):
:param listing: The listing of object segments to iterate over; this may
be an iterator or list that returns dicts with 'name' and
'bytes' keys.
:param response: The webob.Response this iterable is associated with, if
:param response: The swob.Response this iterable is associated with, if
any (default: None)
"""
@ -327,11 +327,11 @@ class ObjectController(Controller):
resp = Response(headers=resp.headers, request=req,
conditional_response=True)
if req.method == 'HEAD':
# These shenanigans are because webob translates the HEAD
# request into a webob EmptyResponse for the body, which
# These shenanigans are because swob translates the HEAD
# request into a swob EmptyResponse for the body, which
# has a len, which eventlet translates as needing a
# content-length header added. So we call the original
# webob resp for the headers but return an empty iterator
# swob resp for the headers but return an empty iterator
# for the body.
def head_response(environ, start_response):

13
swift/proxy/server.py
View File

@ -31,9 +31,6 @@ from ConfigParser import ConfigParser
import uuid
from eventlet import Timeout
from webob.exc import HTTPBadRequest, HTTPForbidden, HTTPMethodNotAllowed, \
HTTPNotFound, HTTPPreconditionFailed, HTTPServerError
from webob import Request
from swift.common.ring import Ring
from swift.common.utils import cache_from_env, get_logger, \
@ -41,6 +38,10 @@ from swift.common.utils import cache_from_env, get_logger, \
from swift.common.constraints import check_utf8
from swift.proxy.controllers import AccountController, ObjectController, \
ContainerController, Controller
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPForbidden, \
HTTPMethodNotAllowed, HTTPNotFound, HTTPPreconditionFailed, \
HTTPRequestEntityTooLarge, HTTPRequestTimeout, HTTPServerError, \
HTTPServiceUnavailable, HTTPClientDisconnect, status_map, Request, Response
class Application(object):
@ -130,7 +131,7 @@ class Application(object):
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in webob.Request object and passes it down.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
@ -157,9 +158,9 @@ class Application(object):
def handle_request(self, req):
"""
Entry point for proxy server.
Should return a WSGI-style callable (such as webob.Response).
Should return a WSGI-style callable (such as swob.Response).
:param req: webob.Request object
:param req: swob.Request object
"""
try:
self.logger.set_statsd_prefix('proxy-server')

6
test/functional/tests.py
View File

@ -1128,7 +1128,7 @@ class TestFile(Base):
range_string = 'bytes=-%d' % (i)
hdrs = {'Range': range_string}
self.assert_(file.read(hdrs=hdrs) == data[-i:], range_string)
self.assertEquals(file.read(hdrs=hdrs), data[-i:])
range_string = 'bytes=%d-' % (i)
hdrs = {'Range': range_string}
@ -1149,10 +1149,6 @@ class TestFile(Base):
def testRangedGetsWithLWSinHeader(self):
#Skip this test until webob 1.2 can tolerate LWS in Range header.
from webob.byterange import Range
if not isinstance(Range.parse('bytes = 0-99 '), Range):
raise SkipTest
file_length = 10000
range_size = file_length / 10
file = self.env.container.file(Utils.create_name())

20
test/unit/account/test_server.py
View File

@ -21,8 +21,8 @@ from StringIO import StringIO
import simplejson
import xml.dom.minidom
from webob import Request
from swift.common.swob import Request
from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT
from swift.common.utils import normalize_timestamp
@ -111,9 +111,9 @@ class TestAccountController(unittest.TestCase):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
resp = self.controller.HEAD(req)
self.assertEquals(resp.status_int, 204)
self.assertEquals(resp.headers['x-account-container-count'], 0)
self.assertEquals(resp.headers['x-account-object-count'], 0)
self.assertEquals(resp.headers['x-account-bytes-used'], 0)
self.assertEquals(resp.headers['x-account-container-count'], '0')
self.assertEquals(resp.headers['x-account-object-count'], '0')
self.assertEquals(resp.headers['x-account-bytes-used'], '0')
def test_HEAD_with_containers(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
@ -136,9 +136,9 @@ class TestAccountController(unittest.TestCase):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
resp = self.controller.HEAD(req)
self.assertEquals(resp.status_int, 204)
self.assertEquals(resp.headers['x-account-container-count'], 2)
self.assertEquals(resp.headers['x-account-object-count'], 0)
self.assertEquals(resp.headers['x-account-bytes-used'], 0)
self.assertEquals(resp.headers['x-account-container-count'], '2')
self.assertEquals(resp.headers['x-account-object-count'], '0')
self.assertEquals(resp.headers['x-account-bytes-used'], '0')
req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Put-Timestamp': '1',
'X-Delete-Timestamp': '0',
@ -157,9 +157,9 @@ class TestAccountController(unittest.TestCase):
'HTTP_X_TIMESTAMP': '5'})
resp = self.controller.HEAD(req)
self.assertEquals(resp.status_int, 204)
self.assertEquals(resp.headers['x-account-container-count'], 2)
self.assertEquals(resp.headers['x-account-object-count'], 4)
self.assertEquals(resp.headers['x-account-bytes-used'], 6)
self.assertEquals(resp.headers['x-account-container-count'], '2')
self.assertEquals(resp.headers['x-account-object-count'], '4')
self.assertEquals(resp.headers['x-account-bytes-used'], '6')
def test_PUT_not_found(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},

3
test/unit/common/middleware/test_cname_lookup.py
View File

@ -16,8 +16,6 @@
import unittest
from nose import SkipTest
from webob import Request
try:
# this test requires the dnspython package to be installed
import dns.resolver
@ -26,6 +24,7 @@ except ImportError:
else: # executed if the try has no errors
skip = False
from swift.common.middleware import cname_lookup
from swift.common.swob import Request
class FakeApp(object):

3
test/unit/common/middleware/test_domain_remap.py
View File

@ -15,8 +15,7 @@
import unittest
from webob import Request
from swift.common.swob import Request
from swift.common.middleware import domain_remap

3
test/unit/common/middleware/test_except.py
View File

@ -15,8 +15,7 @@
import unittest
from webob import Request, Response
from swift.common.swob import Request, Response
from swift.common.middleware import catch_errors
from swift.common.utils import get_logger

3
test/unit/common/middleware/test_formpost.py
View File

@ -20,8 +20,7 @@ from contextlib import contextmanager
from StringIO import StringIO
from time import time
from webob import Request, Response
from swift.common.swob import Request, Response
from swift.common.middleware import tempauth, formpost

3
test/unit/common/middleware/test_healthcheck.py
View File

@ -15,8 +15,7 @@
import unittest
from webob import Request
from swift.common.swob import Request
from swift.common.middleware import healthcheck
class FakeApp(object):

25
test/unit/common/middleware/test_keystoneauth.py
View File

@ -14,9 +14,10 @@
# limitations under the License.
import unittest
import webob
from swift.common.middleware import keystoneauth
from swift.common.swob import Request, Response, HTTPForbidden
from swift.common.http import HTTP_FORBIDDEN
class FakeApp(object):
@ -28,13 +29,13 @@ class FakeApp(object):
def __call__(self, env, start_response):
self.calls += 1
self.request = webob.Request.blank('', environ=env)
self.request = Request.blank('', environ=env)
if 'swift.authorize' in env:
resp = env['swift.authorize'](self.request)
if resp:
return resp(env, start_response)
status, headers, body = self.status_headers_body_iter.next()
return webob.Response(status=status, headers=headers,
return Response(status=status, headers=headers,
body=body)(env, start_response)
@ -45,7 +46,7 @@ class SwiftAuth(unittest.TestCase):
def _make_request(self, path=None, headers=None, **kwargs):
if not path:
path = '/v1/%s/c/o' % self.test_auth._get_account_for_tenant('foo')
return webob.Request.blank(path, headers=headers, **kwargs)
return Request.blank(path, headers=headers, **kwargs)
def _get_identity_headers(self, status='Confirmed', tenant_id='1',
tenant_name='acct', user='usr', role=''):
@ -118,7 +119,7 @@ class TestAuthorize(unittest.TestCase):
self.test_auth = keystoneauth.filter_factory({})(FakeApp())
def _make_request(self, path, **kwargs):
return webob.Request.blank(path, **kwargs)
return Request.blank(path, **kwargs)
def _get_account(self, identity=None):
if not identity:
@ -147,17 +148,17 @@ class TestAuthorize(unittest.TestCase):
req.acl = acl
result = self.test_auth.authorize(req)
if exception:
self.assertTrue(isinstance(result, exception))
self.assertEquals(result.status_int, exception)
else:
self.assertTrue(result is None)
return req
def test_authorize_fails_for_unauthorized_user(self):
self._check_authenticate(exception=webob.exc.HTTPForbidden)
self._check_authenticate(exception=HTTP_FORBIDDEN)
def test_authorize_fails_for_invalid_reseller_prefix(self):
self._check_authenticate(account='BLAN_a',
exception=webob.exc.HTTPForbidden)
exception=HTTP_FORBIDDEN)
def test_authorize_succeeds_for_reseller_admin(self):
roles = [self.test_auth.reseller_admin_role]
@ -185,22 +186,22 @@ class TestAuthorize(unittest.TestCase):
def test_authorize_fails_as_owner_for_tenant_owner_match(self):
self.test_auth.is_admin = False
self._check_authorize_for_tenant_owner_match(
exception=webob.exc.HTTPForbidden)
exception=HTTP_FORBIDDEN)
def test_authorize_succeeds_for_container_sync(self):
env = {'swift_sync_key': 'foo', 'REMOTE_ADDR': '127.0.0.1'}
headers = {'x-container-sync-key': 'foo', 'x-timestamp': None}
headers = {'x-container-sync-key': 'foo', 'x-timestamp': '1'}
self._check_authenticate(env=env, headers=headers)
def test_authorize_fails_for_invalid_referrer(self):
env = {'HTTP_REFERER': 'http://invalid.com/index.html'}
self._check_authenticate(acl='.r:example.com', env=env,
exception=webob.exc.HTTPForbidden)
exception=HTTP_FORBIDDEN)
def test_authorize_fails_for_referrer_without_rlistings(self):
env = {'HTTP_REFERER': 'http://example.com/index.html'}
self._check_authenticate(acl='.r:example.com', env=env,
exception=webob.exc.HTTPForbidden)
exception=HTTP_FORBIDDEN)
def test_authorize_succeeds_for_referrer_with_rlistings(self):
env = {'HTTP_REFERER': 'http://example.com/index.html'}

4
test/unit/common/middleware/test_memcache.py
View File

@ -16,10 +16,10 @@
import unittest
from ConfigParser import NoSectionError, NoOptionError
from webob import Request
from swift.common.middleware import memcache
from swift.common.memcached import MemcacheRing
from swift.common.swob import Request
class FakeApp(object):
def __call__(self, env, start_response):

3
test/unit/common/middleware/test_name_check.py
View File

@ -22,7 +22,8 @@ Created on February 29, 2012
'''
import unittest
from webob import Request, Response
from swift.common.swob import Request, Response
from swift.common.middleware import name_check
MAX_LENGTH = 255

3
test/unit/common/middleware/test_proxy_logging.py
View File

@ -18,11 +18,10 @@ from urllib import quote, unquote
import cStringIO as StringIO
from logging.handlers import SysLogHandler
from webob import Request
from test.unit import FakeLogger
from swift.common.utils import get_logger
from swift.common.middleware import proxy_logging
from swift.common.swob import Request
class FakeApp(object):

2
test/unit/common/middleware/test_ratelimit.py
View File

@ -18,12 +18,12 @@ import time
import eventlet
from contextlib import contextmanager
from threading import Thread
from webob import Request
from test.unit import FakeLogger
from swift.common.middleware import ratelimit
from swift.proxy.controllers.base import get_container_memcache_key
from swift.common.memcached import MemcacheConnectionError
from swift.common.swob import Request
class FakeMemcache(object):

5
test/unit/common/middleware/test_recon.py
View File

@ -14,13 +14,14 @@
# limitations under the License.
import unittest
from webob import Request
from swift.common.middleware import recon
from unittest import TestCase
from contextlib import contextmanager
from posix import stat_result, statvfs_result
import os
import swift.common.constraints
from swift.common.swob import Request
from swift.common.middleware import recon
class FakeApp(object):

3
test/unit/common/middleware/test_staticweb.py
View File

@ -20,8 +20,7 @@ except ImportError:
import unittest
from contextlib import contextmanager
from webob import Request, Response
from swift.common.swob import Request, Response
from swift.common.middleware import staticweb
from test.unit import FakeLogger

3
test/unit/common/middleware/test_tempauth.py
View File

@ -21,9 +21,8 @@ import unittest
from contextlib import contextmanager
from time import time
from webob import Request, Response
from swift.common.middleware import tempauth as auth
from swift.common.swob import Request, Response
class FakeMemcache(object):

3
test/unit/common/middleware/test_tempurl.py
View File

@ -19,8 +19,7 @@ from hashlib import sha1
from contextlib import contextmanager
from time import time
from webob import Request, Response
from swift.common.swob import Request, Response
from swift.common.middleware import tempauth, tempurl

60
test/unit/common/test_constraints.py
View File

@ -16,10 +16,10 @@
import unittest
from test.unit import MockTrue
from webob import Request
from webob.exc import HTTPBadRequest, HTTPLengthRequired, \
HTTPRequestEntityTooLarge
from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \
HTTPRequestEntityTooLarge, Request
from swift.common.http import HTTP_REQUEST_ENTITY_TOO_LARGE, \
HTTP_BAD_REQUEST, HTTP_LENGTH_REQUIRED
from swift.common import constraints
@ -47,8 +47,8 @@ class TestConstraints(unittest.TestCase):
headers=headers), 'object'), None)
name = 'a' * (constraints.MAX_META_NAME_LENGTH + 1)
headers = {'X-Object-Meta-%s' % name: 'v'}
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
headers=headers), 'object'), HTTPBadRequest))
self.assertEquals(constraints.check_metadata(Request.blank('/',
headers=headers), 'object').status_int, HTTP_BAD_REQUEST)
def test_check_metadata_value_length(self):
value = 'a' * constraints.MAX_META_VALUE_LENGTH
@ -57,8 +57,8 @@ class TestConstraints(unittest.TestCase):
headers=headers), 'object'), None)
value = 'a' * (constraints.MAX_META_VALUE_LENGTH + 1)
headers = {'X-Object-Meta-Name': value}
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
headers=headers), 'object'), HTTPBadRequest))
self.assertEquals(constraints.check_metadata(Request.blank('/',
headers=headers), 'object').status_int, HTTP_BAD_REQUEST)
def test_check_metadata_count(self):
headers = {}
@ -67,8 +67,8 @@ class TestConstraints(unittest.TestCase):
self.assertEquals(constraints.check_metadata(Request.blank('/',
headers=headers), 'object'), None)
headers['X-Object-Meta-Too-Many'] = 'v'
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
headers=headers), 'object'), HTTPBadRequest))
self.assertEquals(constraints.check_metadata(Request.blank('/',
headers=headers), 'object').status_int, HTTP_BAD_REQUEST)
def test_check_metadata_size(self):
headers = {}
@ -92,8 +92,8 @@ class TestConstraints(unittest.TestCase):
headers['X-Object-Meta-%04d%s' %
(x + 1, 'a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \
'v' * constraints.MAX_META_VALUE_LENGTH
self.assert_(isinstance(constraints.check_metadata(Request.blank('/',
headers=headers), 'object'), HTTPBadRequest))
self.assertEquals(constraints.check_metadata(Request.blank('/',
headers=headers), 'object').status_int, HTTP_BAD_REQUEST)
def test_check_object_creation_content_length(self):
headers = {'Content-Length': str(constraints.MAX_FILE_SIZE),
@ -102,17 +102,17 @@ class TestConstraints(unittest.TestCase):
headers=headers), 'object_name'), None)
headers = {'Content-Length': str(constraints.MAX_FILE_SIZE + 1),
'Content-Type': 'text/plain'}
self.assert_(isinstance(constraints.check_object_creation(
Request.blank('/', headers=headers), 'object_name'),
HTTPRequestEntityTooLarge))
self.assertEquals(constraints.check_object_creation(
Request.blank('/', headers=headers), 'object_name').status_int,
HTTP_REQUEST_ENTITY_TOO_LARGE)
headers = {'Transfer-Encoding': 'chunked',
'Content-Type': 'text/plain'}
self.assertEquals(constraints.check_object_creation(Request.blank('/',
headers=headers), 'object_name'), None)
headers = {'Content-Type': 'text/plain'}
self.assert_(isinstance(constraints.check_object_creation(
Request.blank('/', headers=headers), 'object_name'),
HTTPLengthRequired))
self.assertEquals(constraints.check_object_creation(
Request.blank('/', headers=headers), 'object_name').status_int,
HTTP_LENGTH_REQUIRED)
def test_check_object_creation_name_length(self):
headers = {'Transfer-Encoding': 'chunked',
@ -121,9 +121,9 @@ class TestConstraints(unittest.TestCase):
self.assertEquals(constraints.check_object_creation(Request.blank('/',
headers=headers), name), None)
name = 'o' * (constraints.MAX_OBJECT_NAME_LENGTH + 1)
self.assert_(isinstance(constraints.check_object_creation(
Request.blank('/', headers=headers), name),
HTTPBadRequest))
self.assertEquals(constraints.check_object_creation(
Request.blank('/', headers=headers), name).status_int,
HTTP_BAD_REQUEST)
def test_check_object_creation_content_type(self):
headers = {'Transfer-Encoding': 'chunked',
@ -131,16 +131,16 @@ class TestConstraints(unittest.TestCase):
self.assertEquals(constraints.check_object_creation(Request.blank('/',
headers=headers), 'object_name'), None)
headers = {'Transfer-Encoding': 'chunked'}
self.assert_(isinstance(constraints.check_object_creation(
Request.blank('/', headers=headers), 'object_name'),
HTTPBadRequest))
self.assertEquals(constraints.check_object_creation(
Request.blank('/', headers=headers), 'object_name').status_int,
HTTP_BAD_REQUEST)
def test_check_object_creation_bad_content_type(self):
headers = {'Transfer-Encoding': 'chunked',
'Content-Type': '\xff\xff'}
resp = constraints.check_object_creation(
Request.blank('/', headers=headers), 'object_name')
self.assert_(isinstance(resp, HTTPBadRequest))
self.assertEquals(resp.status_int, HTTP_BAD_REQUEST)
self.assert_('Content-Type' in resp.body)
def test_check_object_manifest_header(self):
@ -151,23 +151,23 @@ class TestConstraints(unittest.TestCase):
resp = constraints.check_object_creation(Request.blank('/',
headers={'X-Object-Manifest': 'container', 'Content-Length': '0',
'Content-Type': 'text/plain'}), 'manifest')
self.assert_(isinstance(resp, HTTPBadRequest))
self.assertEquals(resp.status_int, HTTP_BAD_REQUEST)
resp = constraints.check_object_creation(Request.blank('/',
headers={'X-Object-Manifest': '/container/prefix',
'Content-Length': '0', 'Content-Type': 'text/plain'}), 'manifest')
self.assert_(isinstance(resp, HTTPBadRequest))
self.assertEquals(resp.status_int, HTTP_BAD_REQUEST)
resp = constraints.check_object_creation(Request.blank('/',
headers={'X-Object-Manifest': 'container/prefix?query=param',
'Content-Length': '0', 'Content-Type': 'text/plain'}), 'manifest')
self.assert_(isinstance(resp, HTTPBadRequest))
self.assertEquals(resp.status_int, HTTP_BAD_REQUEST)
resp = constraints.check_object_creation(Request.blank('/',
headers={'X-Object-Manifest': 'container/prefix&query=param',
'Content-Length': '0', 'Content-Type': 'text/plain'}), 'manifest')
self.assert_(isinstance(resp, HTTPBadRequest))
self.assertEquals(resp.status_int, HTTP_BAD_REQUEST)
resp = constraints.check_object_creation(Request.blank('/',
headers={'X-Object-Manifest': 'http://host/container/prefix',
'Content-Length': '0', 'Content-Type': 'text/plain'}), 'manifest')
self.assert_(isinstance(resp, HTTPBadRequest))
self.assertEquals(resp.status_int, HTTP_BAD_REQUEST)
def test_check_mount(self):
self.assertFalse(constraints.check_mount('', ''))

398
test/unit/common/test_swob.py Normal file
View File

@ -0,0 +1,398 @@
# Copyright (c) 2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"Tests for swift.common.swob"
import unittest
import datetime
import swift.common.swob
class TestHeaderEnvironProxy(unittest.TestCase):
def test_proxy(self):
environ = {}
proxy = swift.common.swob.HeaderEnvironProxy(environ)
proxy['Content-Length'] = 20
proxy['Content-Type'] = 'text/plain'
proxy['Something-Else'] = 'somevalue'
self.assertEquals(
proxy.environ, {'CONTENT_LENGTH': '20',
'CONTENT_TYPE': 'text/plain',
'HTTP_SOMETHING_ELSE': 'somevalue'})
self.assertEquals(proxy['content-length'], '20')
self.assertEquals(proxy['content-type'], 'text/plain')
self.assertEquals(proxy['something-else'], 'somevalue')
def test_del(self):
environ = {}
proxy = swift.common.swob.HeaderEnvironProxy(environ)
proxy['Content-Length'] = 20
proxy['Content-Type'] = 'text/plain'
proxy['Something-Else'] = 'somevalue'
del proxy['Content-Length']
del proxy['Content-Type']
del proxy['Something-Else']
self.assertEquals(proxy.environ, {})
def test_contains(self):
environ = {}
proxy = swift.common.swob.HeaderEnvironProxy(environ)
proxy['Content-Length'] = 20
proxy['Content-Type'] = 'text/plain'
proxy['Something-Else'] = 'somevalue'
self.assert_('content-length' in proxy)
self.assert_('content-type' in proxy)
self.assert_('something-else' in proxy)
def test_keys(self):
environ = {}
proxy = swift.common.swob.HeaderEnvironProxy(environ)
proxy['Content-Length'] = 20
proxy['Content-Type'] = 'text/plain'
proxy['Something-Else'] = 'somevalue'
self.assertEquals(
set(proxy.keys()),
set(('Content-Length', 'Content-Type', 'Something-Else')))
class TestHeaderKeyDict(unittest.TestCase):
def test_case_insensitive(self):
headers = swift.common.swob.HeaderKeyDict()
headers['Content-Length'] = 0
headers['CONTENT-LENGTH'] = 10
headers['content-length'] = 20
self.assertEquals(headers['Content-Length'], '20')
self.assertEquals(headers['content-length'], '20')
self.assertEquals(headers['CONTENT-LENGTH'], '20')
def test_del_contains(self):
headers = swift.common.swob.HeaderKeyDict()
headers['Content-Length'] = 0
self.assert_('Content-Length' in headers)
del headers['Content-Length']
self.assert_('Content-Length' not in headers)
def test_update(self):
headers = swift.common.swob.HeaderKeyDict()
headers.update({'Content-Length': '0'})
headers.update([('Content-Type', 'text/plain')])
self.assertEquals(headers['Content-Length'], '0')
self.assertEquals(headers['Content-Type'], 'text/plain')
def test_get(self):
headers = swift.common.swob.HeaderKeyDict()
headers['content-length'] = 20
self.assertEquals(headers.get('CONTENT-LENGTH'), '20')
self.assertEquals(headers.get('something-else'), None)
self.assertEquals(headers.get('something-else', True), True)
class TestRange(unittest.TestCase):
def test_range(self):
range = swift.common.swob.Range('bytes=1-7')
self.assertEquals(range.ranges[0], (1, 7))
def test_upsidedown_range(self):
range = swift.common.swob.Range('bytes=5-10')
self.assertEquals(range.range_for_length(2), None)
def test_str(self):
for range_str in ('bytes=1-7', 'bytes=1-', 'bytes=-1',
'bytes=1-7,9-12', 'bytes=-7,9-'):
range = swift.common.swob.Range(range_str)
self.assertEquals(str(range), range_str)
def test_range_for_length(self):
range = swift.common.swob.Range('bytes=1-7')
self.assertEquals(range.range_for_length(10), (1, 8))
self.assertEquals(range.range_for_length(5), (1, 5))
self.assertEquals(range.range_for_length(None), None)
def test_range_for_length_no_end(self):
range = swift.common.swob.Range('bytes=1-')
self.assertEquals(range.range_for_length(10), (1, 10))
self.assertEquals(range.range_for_length(5), (1, 5))
self.assertEquals(range.range_for_length(None), None)
def test_range_for_length_no_start(self):
range = swift.common.swob.Range('bytes=-7')
self.assertEquals(range.range_for_length(10), (3, 10))
self.assertEquals(range.range_for_length(5), None)
self.assertEquals(range.range_for_length(None), None)
class TestMatch(unittest.TestCase):
def test_match(self):
match = swift.common.swob.Match('"a", "b"')
self.assertEquals(match.tags, set(('a', 'b')))
self.assert_('a' in match)
self.assert_('b' in match)
self.assert_('c' not in match)
def test_match_star(self):
match = swift.common.swob.Match('"a", "*"')
self.assert_('a' in match)
self.assert_('b' in match)
self.assert_('c' in match)
def test_match_noquote(self):
match = swift.common.swob.Match('a, b')
self.assertEquals(match.tags, set(('a', 'b')))
self.assert_('a' in match)
self.assert_('b' in match)
self.assert_('c' not in match)
class TestAccept(unittest.TestCase):
def test_accept_json(self):
for accept in ('application/json', 'application/json;q=1.0,*/*;q=0.9',
'*/*;q=0.9,application/json;q=1.0', 'application/*'):
acc = swift.common.swob.Accept(accept)
match = acc.best_match(['text/plain', 'application/json',
'application/xml', 'text/xml'],
default_match='text/plain')
self.assertEquals(match, 'application/json')
def test_accept_plain(self):
for accept in ('', 'text/plain', 'application/xml;q=0.8,*/*;q=0.9',
'*/*;q=0.9,application/xml;q=0.8', '*/*',
'text/plain,application/xml'):
acc = swift.common.swob.Accept(accept)
match = acc.best_match(['text/plain', 'application/json',
'application/xml', 'text/xml'],
default_match='text/plain')
self.assertEquals(match, 'text/plain')
def test_accept_xml(self):
for accept in ('application/xml', 'application/xml;q=1.0,*/*;q=0.9',
'*/*;q=0.9,application/xml;q=1.0'):
acc = swift.common.swob.Accept(accept)
match = acc.best_match(['text/plain', 'application/xml',
'text/xml'], default_match='text/plain')
self.assertEquals(match, 'application/xml')
class TestRequest(unittest.TestCase):
def test_blank(self):
req = swift.common.swob.Request.blank(
'/', environ={'REQUEST_METHOD': 'POST'},
headers={'Content-Type': 'text/plain'}, body='hi')
self.assertEquals(req.path_info, '/')
self.assertEquals(req.body, 'hi')
self.assertEquals(req.headers['Content-Type'], 'text/plain')
self.assertEquals(req.method, 'POST')
def test_params(self):
req = swift.common.swob.Request.blank('/?a=b&c=d')
self.assertEquals(req.params['a'], 'b')
self.assertEquals(req.params['c'], 'd')
def test_path(self):
req = swift.common.swob.Request.blank('/hi?a=b&c=d')
self.assertEquals(req.path, '/hi')
req = swift.common.swob.Request.blank(
'/', environ={'SCRIPT_NAME': '/hi', 'PATH_INFO': '/there'})
self.assertEquals(req.path, '/hi/there')
def test_path_info_pop(self):
req = swift.common.swob.Request.blank('/hi/there')
self.assertEquals(req.path_info_pop(), 'hi')
self.assertEquals(req.path_info, '/there')
self.assertEquals(req.script_name, '/hi')
def test_bad_path_info_pop(self):
req = swift.common.swob.Request.blank('blahblah')
self.assertEquals(req.path_info_pop(), None)
def test_copy_get(self):
req = swift.common.swob.Request.blank('/hi/there',
environ={'REQUEST_METHOD': 'POST'})
self.assertEquals(req.method, 'POST')
req2 = req.copy_get()
self.assertEquals(req2.method, 'GET')
def test_get_response(self):
def test_app(environ, start_response):
start_response('200 OK', [])
return ['hi']
req = swift.common.swob.Request.blank('/')
resp = req.get_response(test_app)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.body, 'hi')
def test_properties(self):
req = swift.common.swob.Request.blank('/hi/there', body='hi')
self.assertEquals(req.body, 'hi')
self.assertEquals(req.content_length, 2)
req.remote_addr = 'something'
self.assertEquals(req.environ['REMOTE_ADDR'], 'something')
req.body = 'whatever'
self.assertEquals(req.content_length, 8)
self.assertEquals(req.body, 'whatever')
self.assertEquals(req.method, 'GET')
req.range = 'bytes=1-7'
self.assertEquals(req.range.ranges[0], (1, 7))
self.assert_('Range' in req.headers)
req.range = None
self.assert_('Range' not in req.headers)
def test_datetime_properties(self):
req = swift.common.swob.Request.blank('/hi/there', body='hi')
req.if_unmodified_since = 0
self.assert_(isinstance(req.if_unmodified_since, datetime.datetime))
if_unmodified_since = req.if_unmodified_since
req.if_unmodified_since = if_unmodified_since
self.assertEquals(if_unmodified_since, req.if_unmodified_since)
req.if_unmodified_since = 'something'
self.assertEquals(req.headers['If-Unmodified-Since'], 'something')
self.assertEquals(req.if_unmodified_since, None)
req.if_unmodified_since = -1
self.assertRaises(ValueError, lambda: req.if_unmodified_since)
self.assert_('If-Unmodified-Since' in req.headers)
req.if_unmodified_since = None
self.assert_('If-Unmodified-Since' not in req.headers)
def test_bad_range(self):
req = swift.common.swob.Request.blank('/hi/there', body='hi')
req.range = 'bad range'
self.assertEquals(req.range, None)
class TestStatusMap(unittest.TestCase):
def test_status_map(self):
response_args = []
def start_response(status, headers):
response_args.append(status)
response_args.append(headers)
resp_cls = swift.common.swob.status_map[404]
resp = resp_cls()
self.assertEquals(resp.status_int, 404)
self.assertEquals(resp.title, 'Not Found')
body = ''.join(resp({}, start_response))
self.assert_('The resource could not be found.' in body)
self.assertEquals(response_args[0], '404 Not Found')
headers = dict(response_args[1])
self.assertEquals(headers['content-type'], 'text/html')
self.assert_(int(headers['content-length']) > 0)
class TestResponse(unittest.TestCase):
def _get_response(self):
def test_app(environ, start_response):
start_response('200 OK', [])
return ['hi']
req = swift.common.swob.Request.blank('/')
return req.get_response(test_app)
def test_properties(self):
resp = self._get_response()
resp.location = 'something'
self.assertEquals(resp.location, 'something')
self.assert_('Location' in resp.headers)
resp.location = None
self.assert_('Location' not in resp.headers)
resp.content_type = 'text/plain'
self.assert_('Content-Type' in resp.headers)
resp.content_type = None
self.assert_('Content-Type' not in resp.headers)
def test_unicode_body(self):
resp = self._get_response()
resp.body = u'\N{SNOWMAN}'
self.assertEquals(resp.body, u'\N{SNOWMAN}'.encode('utf-8'))
def test_location_rewrite(self):
def start_response(env, headers):
pass
req = swift.common.swob.Request.blank('/')
resp = self._get_response()
resp.location = '/something'
body = ''.join(resp(req.environ, start_response))
self.assertEquals(resp.location, 'http://localhost/something')
def test_app_iter(self):
def start_response(env, headers):
pass
resp = self._get_response()
resp.app_iter = ['a', 'b', 'c']
body = ''.join(resp({}, start_response))
self.assertEquals(body, 'abc')
def test_range_body(self):
def test_app(environ, start_response):
start_response('200 OK', [('Content-Length', '10')])
return ['1234567890']
def start_response(env, headers):
pass
req = swift.common.swob.Request.blank(
'/', headers={'Range': 'bytes=1-3'})
resp = req.get_response(test_app)
resp.conditional_response = True
body = ''.join(resp([], start_response))
self.assertEquals(body, '234')
resp = swift.common.swob.Response(
body='1234567890', request=req,
conditional_response=True)
body = ''.join(resp([], start_response))
self.assertEquals(body, '234')
def test_content_type(self):
resp = self._get_response()
resp.content_type = 'text/plain; charset=utf8'
self.assertEquals(resp.content_type, 'text/plain')
def test_charset(self):
resp = self._get_response()
resp.content_type = 'text/plain; charset=utf8'
self.assertEquals(resp.charset, 'utf8')
resp.charset = 'utf16'
self.assertEquals(resp.charset, 'utf16')
def test_etag(self):
resp = self._get_response()
resp.etag = 'hi'
self.assertEquals(resp.headers['Etag'], '"hi"')
self.assertEquals(resp.etag, 'hi')
self.assert_('etag' in resp.headers)
resp.etag = None
self.assert_('etag' not in resp.headers)
class TestUTC(unittest.TestCase):
def test_tzname(self):
self.assertEquals(swift.common.swob.UTC.tzname(None), 'UTC')
if __name__ == '__main__':
unittest.main()

2
test/unit/common/test_wsgi.py
View File

@ -29,8 +29,8 @@ from collections import defaultdict
from urllib import quote
from eventlet import sleep
from webob import Request
from swift.common.swob import Request
from swift.common import wsgi
class TestWSGI(unittest.TestCase):

2
test/unit/container/test_server.py
View File

@ -23,8 +23,8 @@ from tempfile import mkdtemp
from eventlet import spawn, Timeout, listen
import simplejson
from webob import Request
from swift.common.swob import Request
from swift.container import server as container_server
from swift.common.utils import normalize_timestamp, mkdirs

2
test/unit/obj/test_internal_client.py
View File

@ -166,7 +166,7 @@ class TestInternalClient(unittest.TestCase):
def fake_app(self, env, start_response):
self.test.assertEquals(self.user_agent, env['HTTP_USER_AGENT'])
start_response('200 Ok', [{'Content-Length': '0'}])
start_response('200 Ok', [('Content-Length', '0')])
return []
client = InternalClient(self)

24
test/unit/obj/test_server.py
View File

@ -25,18 +25,18 @@ from tempfile import mkdtemp
from hashlib import md5
from eventlet import sleep, spawn, wsgi, listen, Timeout
from webob import Request
from test.unit import FakeLogger
from test.unit import _getxattr as getxattr
from test.unit import _setxattr as setxattr
from test.unit import connect_tcp, readuntil2crlfs
from swift.obj import server as object_server
from swift.obj import server as object_server, replicator
from swift.common import utils
from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \
NullLogger, storage_directory
from swift.common.exceptions import DiskFileNotExist
from swift.common import constraints
from eventlet import tpool
from swift.common.swob import Request
class TestDiskFile(unittest.TestCase):
@ -1047,7 +1047,7 @@ class TestObjectController(unittest.TestCase):
resp = self.object_controller.GET(req)
self.assertEquals(resp.status_int, 200)
since = strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp)))
since = strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) + 1))
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'},
headers={'If-Modified-Since': since})
resp = self.object_controller.GET(req)
@ -1521,6 +1521,24 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.headers.get('x-object-manifest'), 'c/o/')
def test_manifest_head_request(self):
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
'Content-Type': 'text/plain',
'Content-Length': '0',
'X-Object-Manifest': 'c/o/'})
req.body = 'hi'
resp = self.object_controller.PUT(req)
objfile = os.path.join(self.testdir, 'sda1',
storage_directory(object_server.DATADIR, 'p', hash_path('a', 'c',
'o')), timestamp + '.data')
self.assert_(os.path.isfile(objfile))
req = Request.blank('/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD'})
resp = self.object_controller.GET(req)
self.assertEquals(resp.body, '')
def test_async_update_http_connect(self):
given_args = []

5
test/unit/proxy/test_server.py
View File

@ -35,8 +35,6 @@ from tempfile import mkdtemp
import eventlet
from eventlet import sleep, spawn, Timeout, util, wsgi, listen
import simplejson
from webob import Request, Response
from webob.exc import HTTPNotFound, HTTPUnauthorized
from test.unit import connect_tcp, readuntil2crlfs, FakeLogger
from swift.proxy import server as proxy_server
@ -54,6 +52,8 @@ from swift.proxy.controllers.obj import SegmentedIterable
from swift.proxy.controllers.base import get_container_memcache_key, \
get_account_memcache_key
import swift.proxy.controllers
from swift.common.swob import Request, Response, HTTPNotFound, \
HTTPUnauthorized
# mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
@ -2742,6 +2742,7 @@ class TestObjectController(unittest.TestCase):
# Do it again but exceeding the container listing limit
swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 2
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
'Host: localhost\r\n'

1
tools/pip-requires
View File

@ -1,4 +1,3 @@
WebOb>=1.0.8,<1.3
configobj==4.7.1
eventlet==0.9.15
greenlet==0.3.1