openstack/swift
Code Issues Proposed changes

Refactor server side copy as middleware

Browse Source 
Rewrite server side copy and 'object post as copy' feature as middleware to
simplify the PUT method in the object controller code. COPY is no longer
a verb implemented as public method in Proxy application.

The server side copy middleware is inserted to the left of dlo, slo and
versioned_writes middlewares in the proxy server pipeline. As a result,
dlo and slo copy_hooks are no longer required. SLO manifests are now
validated when copied so when copying a manifest to another account the
referenced segments must be readable in that account for the manifest
copy to succeed (previously this validation was not made, meaning the
manifest was copied but could be unusable if the segments were not
readable).

With this change, there should be no change in functionality or existing
behavior. This is asserted with (almost) no changes required to existing
functional tests.

Some notes (for operators):
* Middleware required to be auto-inserted before slo and dlo and
  versioned_writes
* Turning off server side copy is not configurable.
* object_post_as_copy is no longer a configurable option of proxy server
  but of this middleware. However, for smooth upgrade, config option set
  in proxy server app is also read.

DocImpact: Introducing server side copy as middleware

Co-Authored-By: Alistair Coles <alistair.coles@hpe.com>
Co-Authored-By: Thiago da Silva <thiago@redhat.com>

Change-Id: Ic96a92e938589a2f6add35a40741fd062f1c29eb
Signed-off-by: Prashanth Pai <ppai@redhat.com>
Signed-off-by: Thiago da Silva <thiago@redhat.com>
This commit is contained in:
Prashanth Pai 2015-02-18 11:59:31 +05:30 committed by Thiago da Silva
parent 72372c1464
commit 46d61a4dcd
30 changed files with 2301 additions and 2314 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
doc/saio/swift/proxy-server.conf
View File

@ -9,7 +9,7 @@ eventlet_debug = true
[pipeline:main]
# Yes, proxy-logging appears twice. This is so that
# middleware-originated requests get logged too.
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain container_sync tempauth staticweb container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain container_sync tempauth staticweb copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
[filter:catch_errors]
use = egg:swift#catch_errors
@ -68,6 +68,9 @@ use = egg:swift#gatekeeper
use = egg:swift#versioned_writes
allow_versioned_writes = true
[filter:copy]
use = egg:swift#copy
[app:proxy-server]
use = egg:swift#proxy
allow_account_management = true

1
doc/source/logs.rst
View File

@ -103,6 +103,7 @@ LE :ref:`list_endpoints`
KS :ref:`keystoneauth`
RL :ref:`ratelimit`
VW :ref:`versioned_writes`
SSC :ref:`copy`
======================= =============================

9
doc/source/middleware.rst
View File

@ -187,6 +187,15 @@ Recon
:members:
:show-inheritance:
.. _copy:
Server Side Copy
================
.. automodule:: swift.common.middleware.copy
:members:
:show-inheritance:
Static Large Objects
====================

20
etc/proxy-server.conf-sample
View File

@ -79,12 +79,12 @@ bind_port = 8080
[pipeline:main]
# This sample pipeline uses tempauth and is used for SAIO dev work and
# testing. See below for a pipeline using keystone.
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
# The following pipeline shows keystone integration. Comment out the one
# above and uncomment this one. Additional steps for integrating keystone are
# covered further below in the filter sections for authtoken and keystoneauth.
#pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit authtoken keystoneauth container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
#pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit authtoken keystoneauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
[app:proxy-server]
use = egg:swift#proxy
@ -129,11 +129,6 @@ use = egg:swift#proxy
# 'false' no one, even authorized, can.
# allow_account_management = false
#
# Set object_post_as_copy = false to turn on fast posts where only the metadata
# changes are stored anew and the original data file is kept in place. This
# makes for quicker posts.
# object_post_as_copy = true
#
# If set to 'true' authorized accounts that do not yet exist within the Swift
# cluster will be automatically created.
# account_autocreate = false
@ -749,3 +744,14 @@ use = egg:swift#versioned_writes
# in the container configuration file, which will be eventually
# deprecated. See documentation for more details.
# allow_versioned_writes = false
# Note: Put after auth and before dlo and slo middlewares.
# If you don't put it in the pipeline, it will be inserted for you.
[filter:copy]
use = egg:swift#copy
# Set object_post_as_copy = false to turn on fast posts where only the metadata
# changes are stored anew and the original data file is kept in place. This
# makes for quicker posts.
# When object_post_as_copy is set to True, a POST request will be transformed
# into a COPY request where source and destination objects are the same.
# object_post_as_copy = true

1
setup.cfg
View File

@ -96,6 +96,7 @@ paste.filter_factory =
container_sync = swift.common.middleware.container_sync:filter_factory
xprofile = swift.common.middleware.xprofile:filter_factory
versioned_writes = swift.common.middleware.versioned_writes:filter_factory
copy = swift.common.middleware.copy:filter_factory
[build_sphinx]
all_files = 1

62
swift/common/constraints.py
View File

@ -20,7 +20,6 @@ import time
import six
from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
from six.moves import urllib
from six.moves.urllib.parse import unquote
from swift.common import utils, exceptions
from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \
@ -205,10 +204,6 @@ def check_object_creation(req, object_name):
request=req,
content_type='text/plain')
if 'X-Copy-From' in req.headers and req.content_length:
return HTTPBadRequest(body='Copy requests require a zero byte body',
request=req, content_type='text/plain')
if len(object_name) > MAX_OBJECT_NAME_LENGTH:
return HTTPBadRequest(body='Object name length of %d longer than %d' %
(len(object_name), MAX_OBJECT_NAME_LENGTH),
@ -359,63 +354,6 @@ def check_utf8(string):
return False
def check_path_header(req, name, length, error_msg):
"""
Validate that the value of path-like header is
well formatted. We assume the caller ensures that
specific header is present in req.headers.
:param req: HTTP request object
:param name: header name
:param length: length of path segment check
:param error_msg: error message for client
:returns: A tuple with path parts according to length
:raise: HTTPPreconditionFailed if header value
is not well formatted.
"""
src_header = unquote(req.headers.get(name))
if not src_header.startswith('/'):
src_header = '/' + src_header
try:
return utils.split_path(src_header, length, length, True)
except ValueError:
raise HTTPPreconditionFailed(
request=req,
body=error_msg)
def check_copy_from_header(req):
"""
Validate that the value from x-copy-from header is
well formatted. We assume the caller ensures that
x-copy-from header is present in req.headers.
:param req: HTTP request object
:returns: A tuple with container name and object name
:raise: HTTPPreconditionFailed if x-copy-from value
is not well formatted.
"""
return check_path_header(req, 'X-Copy-From', 2,
'X-Copy-From header must be of the form '
'<container name>/<object name>')
def check_destination_header(req):
"""
Validate that the value from destination header is
well formatted. We assume the caller ensures that
destination header is present in req.headers.
:param req: HTTP request object
:returns: A tuple with container name and object name
:raise: HTTPPreconditionFailed if destination value
is not well formatted.
"""
return check_path_header(req, 'Destination', 2,
'Destination header must be of the form '
'<container name>/<object name>')
def check_name_format(req, name, target_type):
"""
Validate that the header contains valid account or container name.

22
swift/common/middleware/account_quotas.py
View File

@ -52,11 +52,10 @@ Due to the eventual consistency further uploads might be possible until the
account size has been updated.
"""
from swift.common.constraints import check_copy_from_header
from swift.common.swob import HTTPForbidden, HTTPBadRequest, \
HTTPRequestEntityTooLarge, wsgify
from swift.common.utils import register_swift_info
from swift.proxy.controllers.base import get_account_info, get_object_info
from swift.proxy.controllers.base import get_account_info
class AccountQuotaMiddleware(object):
@ -71,7 +70,7 @@ class AccountQuotaMiddleware(object):
@wsgify
def __call__(self, request):
if request.method not in ("POST", "PUT", "COPY"):
if request.method not in ("POST", "PUT"):
return self.app
try:
@ -106,15 +105,6 @@ class AccountQuotaMiddleware(object):
if request.method == "POST" or not obj:
return self.app
if request.method == 'COPY':
copy_from = container + '/' + obj
else:
if 'x-copy-from' in request.headers:
src_cont, src_obj = check_copy_from_header(request)
copy_from = "%s/%s" % (src_cont, src_obj)
else:
copy_from = None
content_length = (request.content_length or 0)
account_info = get_account_info(request.environ, self.app)
@ -127,14 +117,6 @@ class AccountQuotaMiddleware(object):
if quota < 0:
return self.app
if copy_from:
path = '/' + ver + '/' + account + '/' + copy_from
object_info = get_object_info(request.environ, self.app, path)
if not object_info or not object_info['length']:
content_length = 0
else:
content_length = int(object_info['length'])
new_size = int(account_info['bytes']) + content_length
if quota < new_size:
resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.')

36
swift/common/middleware/container_quotas.py
View File

@ -51,13 +51,11 @@ For example::
[filter:container_quotas]
use = egg:swift#container_quotas
"""
from swift.common.constraints import check_copy_from_header, \
check_account_format, check_destination_header
from swift.common.http import is_success
from swift.common.swob import HTTPRequestEntityTooLarge, HTTPBadRequest, \
wsgify
from swift.common.utils import register_swift_info
from swift.proxy.controllers.base import get_container_info, get_object_info
from swift.proxy.controllers.base import get_container_info
class ContainerQuotaMiddleware(object):
@ -91,25 +89,9 @@ class ContainerQuotaMiddleware(object):
return HTTPBadRequest(body='Invalid count quota.')
# check user uploads against quotas
elif obj and req.method in ('PUT', 'COPY'):
container_info = None
if req.method == 'PUT':
container_info = get_container_info(
req.environ, self.app, swift_source='CQ')
if req.method == 'COPY' and 'Destination' in req.headers:
dest_account = account
if 'Destination-Account' in req.headers:
dest_account = req.headers.get('Destination-Account')
dest_account = check_account_format(req, dest_account)
dest_container, dest_object = check_destination_header(req)
path_info = req.environ['PATH_INFO']
req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
version, dest_account, dest_container, dest_object)
try:
container_info = get_container_info(
req.environ, self.app, swift_source='CQ')
finally:
req.environ['PATH_INFO'] = path_info
elif obj and req.method in ('PUT'):
container_info = get_container_info(
req.environ, self.app, swift_source='CQ')
if not container_info or not is_success(container_info['status']):
# this will hopefully 404 later
return self.app
@ -118,16 +100,6 @@ class ContainerQuotaMiddleware(object):
'bytes' in container_info and \
container_info['meta']['quota-bytes'].isdigit():
content_length = (req.content_length or 0)
if 'x-copy-from' in req.headers or req.method == 'COPY':
if 'x-copy-from' in req.headers:
container, obj = check_copy_from_header(req)
path = '/%s/%s/%s/%s' % (version, account,
container, obj)
object_info = get_object_info(req.environ, self.app, path)
if not object_info or not object_info['length']:
content_length = 0
else:
content_length = int(object_info['length'])
new_size = int(container_info['bytes']) + content_length
if int(container_info['meta']['quota-bytes']) < new_size:
return self.bad_response(req, container_info)

522
swift/common/middleware/copy.py Normal file
View File

@ -0,0 +1,522 @@
# Copyright (c) 2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Server side copy is a feature that enables users/clients to COPY objects
between accounts and containers without the need to download and then
re-upload objects, thus eliminating additional bandwidth consumption and
also saving time. This may be used when renaming/moving an object which
in Swift is a (COPY + DELETE) operation.
The server side copy middleware should be inserted in the pipeline after auth
and before the quotas and large object middlewares. If it is not present in the
pipeline in the proxy-server configuration file, it will be inserted
automatically. There is no configurable option provided to turn off server
side copy.
--------
Metadata
--------
* All metadata of source object is preserved during object copy.
* One can also provide additional metadata during PUT/COPY request. This will
over-write any existing conflicting keys.
* Server side copy can also be used to change content-type of an existing
object.
-----------
Object Copy
-----------
* The destination container must exist before requesting copy of the object.
* When several replicas exist, the system copies from the most recent replica.
That is, the copy operation behaves as though the X-Newest header is in the
request.
* The request to copy an object should have no body (i.e. content-length of the
request must be zero).
There are two ways in which an object can be copied:
1. Send a PUT request to the new object (destination/target) with an additional
header named ``X-Copy-From`` specifying the source object
(in '/container/object' format). Example::
curl -i -X PUT http://<storage_url>/container1/destination_obj
-H 'X-Auth-Token: <token>'
-H 'X-Copy-From: /container2/source_obj'
-H 'Content-Length: 0'
2. Send a COPY request with an existing object in URL with an additional header
named ``Destination`` specifying the destination/target object
(in '/container/object' format). Example::
curl -i -X COPY http://<storage_url>/container2/source_obj
-H 'X-Auth-Token: <token>'
-H 'Destination: /container1/destination_obj'
-H 'Content-Length: 0'
Note that if the incoming request has some conditional headers (e.g. ``Range``,
``If-Match``), the *source* object will be evaluated for these headers (i.e. if
PUT with both ``X-Copy-From`` and ``Range``, Swift will make a partial copy to
the destination object).
-------------------------
Cross Account Object Copy
-------------------------
Objects can also be copied from one account to another account if the user
has the necessary permissions (i.e. permission to read from container
in source account and permission to write to container in destination account).
Similar to examples mentioned above, there are two ways to copy objects across
accounts:
1. Like the example above, send PUT request to copy object but with an
additional header named ``X-Copy-From-Account`` specifying the source
account. Example::
curl -i -X PUT http://<host>:<port>/v1/AUTH_test1/container/destination_obj
-H 'X-Auth-Token: <token>'
-H 'X-Copy-From: /container/source_obj'
-H 'X-Copy-From-Account: AUTH_test2'
-H 'Content-Length: 0'
2. Like the previous example, send a COPY request but with an additional header
named ``Destination-Account`` specifying the name of destination account.
Example::
curl -i -X COPY http://<host>:<port>/v1/AUTH_test2/container/source_obj
-H 'X-Auth-Token: <token>'
-H 'Destination: /container/destination_obj'
-H 'Destination-Account: AUTH_test1'
-H 'Content-Length: 0'
-------------------
Large Object Copy
-------------------
The best option to copy a large option is to copy segments individually.
To copy the manifest object of a large object, add the query parameter to
the copy request::
?multipart-manifest=get
If a request is sent without the query parameter, an attempt will be made to
copy the whole object but will fail if the object size is
greater than 5GB.
-------------------
Object Post as Copy
-------------------
Historically, this has been a feature (and a configurable option with default
set to True) in proxy server configuration. This has been moved to server side
copy middleware.
When ``object_post_as_copy`` is set to ``true`` (default value), an incoming
POST request is morphed into a COPY request where source and destination
objects are same.
This feature was necessary because of a previous behavior where POSTS would
update the metadata on the object but not on the container. As a result,
features like container sync would not work correctly. This is no longer the
case and the plan is to deprecate this option. It is being kept now for
backwards compatibility. At first chance, set ``object_post_as_copy`` to
``false``.
"""
import os
from urllib import quote
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from six.moves.urllib.parse import unquote
from swift.common import utils
from swift.common.utils import get_logger, \
config_true_value, FileLikeIter, read_conf_dir, close_if_possible
from swift.common.swob import Request, HTTPPreconditionFailed, \
HTTPRequestEntityTooLarge, HTTPBadRequest
from swift.common.http import HTTP_MULTIPLE_CHOICES, HTTP_CREATED, \
is_success
from swift.common.constraints import check_account_format, MAX_FILE_SIZE
from swift.common.request_helpers import copy_header_subset, remove_items, \
is_sys_meta, is_sys_or_user_meta
from swift.common.wsgi import WSGIContext, make_subrequest
def _check_path_header(req, name, length, error_msg):
"""
Validate that the value of path-like header is
well formatted. We assume the caller ensures that
specific header is present in req.headers.
:param req: HTTP request object
:param name: header name
:param length: length of path segment check
:param error_msg: error message for client
:returns: A tuple with path parts according to length
:raise: HTTPPreconditionFailed if header value
is not well formatted.
"""
src_header = unquote(req.headers.get(name))
if not src_header.startswith('/'):
src_header = '/' + src_header
try:
return utils.split_path(src_header, length, length, True)
except ValueError:
raise HTTPPreconditionFailed(
request=req,
body=error_msg)
def _check_copy_from_header(req):
"""
Validate that the value from x-copy-from header is
well formatted. We assume the caller ensures that
x-copy-from header is present in req.headers.
:param req: HTTP request object
:returns: A tuple with container name and object name
:raise: HTTPPreconditionFailed if x-copy-from value
is not well formatted.
"""
return _check_path_header(req, 'X-Copy-From', 2,
'X-Copy-From header must be of the form '
'<container name>/<object name>')
def _check_destination_header(req):
"""
Validate that the value from destination header is
well formatted. We assume the caller ensures that
destination header is present in req.headers.
:param req: HTTP request object
:returns: A tuple with container name and object name
:raise: HTTPPreconditionFailed if destination value
is not well formatted.
"""
return _check_path_header(req, 'Destination', 2,
'Destination header must be of the form '
'<container name>/<object name>')
def _copy_headers_into(from_r, to_r):
"""
Will copy desired headers from from_r to to_r
:params from_r: a swob Request or Response
:params to_r: a swob Request or Response
"""
pass_headers = ['x-delete-at']
for k, v in from_r.headers.items():
if is_sys_or_user_meta('object', k) or k.lower() in pass_headers:
to_r.headers[k] = v
class ServerSideCopyWebContext(WSGIContext):
def __init__(self, app, logger):
super(ServerSideCopyWebContext, self).__init__(app)
self.app = app
self.logger = logger
def get_source_resp(self, req):
sub_req = make_subrequest(
req.environ, path=req.path_info, headers=req.headers,
swift_source='SSC')
return sub_req.get_response(self.app)
def send_put_req(self, req, additional_resp_headers, start_response):
app_resp = self._app_call(req.environ)
self._adjust_put_response(req, additional_resp_headers)
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return app_resp
def _adjust_put_response(self, req, additional_resp_headers):
if 'swift.post_as_copy' in req.environ:
# Older editions returned 202 Accepted on object POSTs, so we'll
# convert any 201 Created responses to that for compatibility with
# picky clients.
if self._get_status_int() == HTTP_CREATED:
self._response_status = '202 Accepted'
elif is_success(self._get_status_int()):
for header, value in additional_resp_headers.items():
self._response_headers.append((header, value))
def handle_OPTIONS_request(self, req, start_response):
app_resp = self._app_call(req.environ)
if is_success(self._get_status_int()):
for i, (header, value) in enumerate(self._response_headers):
if header.lower() == 'allow' and 'COPY' not in value:
self._response_headers[i] = ('Allow', value + ', COPY')
if header.lower() == 'access-control-allow-methods' and \
'COPY' not in value:
self._response_headers[i] = \
('Access-Control-Allow-Methods', value + ', COPY')
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return app_resp
class ServerSideCopyMiddleware(object):
def __init__(self, app, conf):
self.app = app
self.logger = get_logger(conf, log_route="copy")
# Read the old object_post_as_copy option from Proxy app just in case
# someone has set it to false (non default). This wouldn't cause
# problems during upgrade.
self._load_object_post_as_copy_conf(conf)
self.object_post_as_copy = \
config_true_value(conf.get('object_post_as_copy', 'true'))
def _load_object_post_as_copy_conf(self, conf):
if ('object_post_as_copy' in conf or '__file__' not in conf):
# Option is explicitly set in middleware conf. In that case,
# we assume operator knows what he's doing.
# This takes preference over the one set in proxy app
return
cp = ConfigParser()
if os.path.isdir(conf['__file__']):
read_conf_dir(cp, conf['__file__'])
else:
cp.read(conf['__file__'])
try:
pipe = cp.get("pipeline:main", "pipeline")
except (NoSectionError, NoOptionError):
return
proxy_name = pipe.rsplit(None, 1)[-1]
proxy_section = "app:" + proxy_name
try:
conf['object_post_as_copy'] = cp.get(proxy_section,
'object_post_as_copy')
except (NoSectionError, NoOptionError):
pass
def __call__(self, env, start_response):
req = Request(env)
try:
(version, account, container, obj) = req.split_path(4, 4, True)
except ValueError:
# If obj component is not present in req, do not proceed further.
return self.app(env, start_response)
self.account_name = account
self.container_name = container
self.object_name = obj
# Save off original request method (COPY/POST) in case it gets mutated
# into PUT during handling. This way logging can display the method
# the client actually sent.
req.environ['swift.orig_req_method'] = req.method
if req.method == 'PUT' and req.headers.get('X-Copy-From'):
return self.handle_PUT(req, start_response)
elif req.method == 'COPY':
return self.handle_COPY(req, start_response)
elif req.method == 'POST' and self.object_post_as_copy:
return self.handle_object_post_as_copy(req, start_response)
elif req.method == 'OPTIONS':
# Does not interfere with OPTIONS response from (account,container)
# servers and /info response.
return self.handle_OPTIONS(req, start_response)
return self.app(env, start_response)
def handle_object_post_as_copy(self, req, start_response):
req.method = 'PUT'
req.path_info = '/v1/%s/%s/%s' % (
self.account_name, self.container_name, self.object_name)
req.headers['Content-Length'] = 0
req.headers.pop('Range', None)
req.headers['X-Copy-From'] = quote('/%s/%s' % (self.container_name,
self.object_name))
req.environ['swift.post_as_copy'] = True
params = req.params
# for post-as-copy always copy the manifest itself if source is *LO
params['multipart-manifest'] = 'get'
req.params = params
return self.handle_PUT(req, start_response)
def handle_COPY(self, req, start_response):
if not req.headers.get('Destination'):
return HTTPPreconditionFailed(request=req,
body='Destination header required'
)(req.environ, start_response)
dest_account = self.account_name
if 'Destination-Account' in req.headers:
dest_account = req.headers.get('Destination-Account')
dest_account = check_account_format(req, dest_account)
req.headers['X-Copy-From-Account'] = self.account_name
self.account_name = dest_account
del req.headers['Destination-Account']
dest_container, dest_object = _check_destination_header(req)
source = '/%s/%s' % (self.container_name, self.object_name)
self.container_name = dest_container
self.object_name = dest_object
# re-write the existing request as a PUT instead of creating a new one
req.method = 'PUT'
# As this the path info is updated with destination container,
# the proxy server app will use the right object controller
# implementation corresponding to the container's policy type.
ver, _junk = req.split_path(1, 2, rest_with_last=True)
req.path_info = '/%s/%s/%s/%s' % \
(ver, dest_account, dest_container, dest_object)
req.headers['Content-Length'] = 0
req.headers['X-Copy-From'] = quote(source)
del req.headers['Destination']
return self.handle_PUT(req, start_response)
def _get_source_object(self, ssc_ctx, source_path, req):
source_req = req.copy_get()
# make sure the source request uses it's container_info
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = quote(source_path)
source_req.headers['X-Newest'] = 'true'
if 'swift.post_as_copy' in req.environ:
# We're COPYing one object over itself because of a POST; rely on
# the PUT for write authorization, don't require read authorization
source_req.environ['swift.authorize'] = lambda req: None
source_req.environ['swift.authorize_override'] = True
# in case we are copying an SLO manifest, set format=raw parameter
params = source_req.params
if params.get('multipart-manifest') == 'get':
params['format'] = 'raw'
source_req.params = params
source_resp = ssc_ctx.get_source_resp(source_req)
if source_resp.content_length is None:
# This indicates a transfer-encoding: chunked source object,
# which currently only happens because there are more than
# CONTAINER_LISTING_LIMIT segments in a segmented object. In
# this case, we're going to refuse to do the server-side copy.
return HTTPRequestEntityTooLarge(request=req)
if source_resp.content_length > MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
return source_resp
def _create_response_headers(self, source_path, source_resp, sink_req):
resp_headers = dict()
acct, path = source_path.split('/', 3)[2:4]
resp_headers['X-Copied-From-Account'] = quote(acct)
resp_headers['X-Copied-From'] = quote(path)
if 'last-modified' in source_resp.headers:
resp_headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
# Existing sys and user meta of source object is added to response
# headers in addition to the new ones.
for k, v in sink_req.headers.items():
if is_sys_or_user_meta('object', k) or k.lower() == 'x-delete-at':
resp_headers[k] = v
return resp_headers
def handle_PUT(self, req, start_response):
if req.content_length:
return HTTPBadRequest(body='Copy requests require a zero byte '
'body', request=req,
content_type='text/plain')(req.environ,
start_response)
# Form the path of source object to be fetched
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account')
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = _check_copy_from_header(req)
source_path = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
self.logger.info("Copying object from %s to %s" %
(source_path, req.path))
# GET the source object, bail out on error
ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
source_resp = self._get_source_object(ssc_ctx, source_path, req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
close_if_possible(source_resp.app_iter)
return source_resp(source_resp.environ, start_response)
# Create a new Request object based on the original req instance.
# This will preserve env and headers.
sink_req = Request.blank(req.path_info,
environ=req.environ, headers=req.headers)
params = sink_req.params
if params.get('multipart-manifest') == 'get':
if 'X-Static-Large-Object' in source_resp.headers:
params['multipart-manifest'] = 'put'
if 'X-Object-Manifest' in source_resp.headers:
del params['multipart-manifest']
sink_req.headers['X-Object-Manifest'] = \
source_resp.headers['X-Object-Manifest']
sink_req.params = params
# Set data source, content length and etag for the PUT request
sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
sink_req.etag = source_resp.etag
# We no longer need these headers
sink_req.headers.pop('X-Copy-From', None)
sink_req.headers.pop('X-Copy-From-Account', None)
# If the copy request does not explicitly override content-type,
# use the one present in the source object.
if not req.headers.get('content-type'):
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
fresh_meta_flag = config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false'))
if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ:
# Post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# Copy/update existing sysmeta and user meta
_copy_headers_into(source_resp, sink_req)
# Copy/update new metadata provided in request if any
_copy_headers_into(req, sink_req)
# Create response headers for PUT response
resp_headers = self._create_response_headers(source_path,
source_resp, sink_req)
put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
close_if_possible(source_resp.app_iter)
return put_resp
def handle_OPTIONS(self, req, start_response):
return ServerSideCopyWebContext(self.app, self.logger).\
handle_OPTIONS_request(req, start_response)
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def copy_filter(app):
return ServerSideCopyMiddleware(app, conf)
return copy_filter

23
swift/common/middleware/dlo.py
View File

@ -405,11 +405,6 @@ class DynamicLargeObject(object):
except ValueError:
return self.app(env, start_response)
# install our COPY-callback hook
env['swift.copy_hook'] = self.copy_hook(
env.get('swift.copy_hook',
lambda src_req, src_resp, sink_req: src_resp))
if ((req.method == 'GET' or req.method == 'HEAD') and
req.params.get('multipart-manifest') != 'get'):
return GetContext(self, self.logger).\
@ -438,24 +433,6 @@ class DynamicLargeObject(object):
body=('X-Object-Manifest must be in the '
'format container/prefix'))
def copy_hook(self, inner_hook):
def dlo_copy_hook(source_req, source_resp, sink_req):
x_o_m = source_resp.headers.get('X-Object-Manifest')
if x_o_m:
if source_req.params.get('multipart-manifest') == 'get':
# To copy the manifest, we let the copy proceed as normal,
# but ensure that X-Object-Manifest is set on the new
# object.
sink_req.headers['X-Object-Manifest'] = x_o_m
else:
ctx = GetContext(self, self.logger)
source_resp = ctx.get_or_head_response(
source_req, x_o_m, source_resp.headers.items())
return inner_hook(source_req, source_resp, sink_req)
return dlo_copy_hook
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()

19
swift/common/middleware/slo.py
View File

@ -798,20 +798,6 @@ class StaticLargeObject(object):
"""
return SloGetContext(self).handle_slo_get_or_head(req, start_response)
def copy_hook(self, inner_hook):
def slo_hook(source_req, source_resp, sink_req):
x_slo = source_resp.headers.get('X-Static-Large-Object')
if (config_true_value(x_slo)
and source_req.params.get('multipart-manifest') != 'get'
and 'swift.post_as_copy' not in source_req.environ):
source_resp = SloGetContext(self).get_or_head_response(
source_req, source_resp.headers.items(),
source_resp.app_iter)
return inner_hook(source_req, source_resp, sink_req)
return slo_hook
def handle_multipart_put(self, req, start_response):
"""
Will handle the PUT of a SLO manifest.
@ -1058,11 +1044,6 @@ class StaticLargeObject(object):
except ValueError:
return self.app(env, start_response)
# install our COPY-callback hook
env['swift.copy_hook'] = self.copy_hook(
env.get('swift.copy_hook',
lambda src_req, src_resp, sink_req: src_resp))
try:
if req.method == 'PUT' and \
req.params.get('multipart-manifest') == 'put':

32
swift/common/middleware/versioned_writes.py
View File

@ -127,9 +127,7 @@ from swift.common.request_helpers import get_sys_meta_prefix, \
from swift.common.wsgi import WSGIContext, make_pre_authed_request
from swift.common.swob import (
Request, HTTPException, HTTPRequestEntityTooLarge)
from swift.common.constraints import (
check_account_format, check_container_format, check_destination_header,
MAX_FILE_SIZE)
from swift.common.constraints import check_container_format, MAX_FILE_SIZE
from swift.proxy.controllers.base import get_container_info
from swift.common.http import (
is_success, is_client_error, HTTP_NOT_FOUND)
@ -493,24 +491,10 @@ class VersionedWritesMiddleware(object):
account_name = unquote(account)
container_name = unquote(container)
object_name = unquote(obj)
container_info = None
resp = None
is_enabled = config_true_value(allow_versioned_writes)
if req.method in ('PUT', 'DELETE'):
container_info = get_container_info(
req.environ, self.app)
elif req.method == 'COPY' and 'Destination' in req.headers:
if 'Destination-Account' in req.headers:
account_name = req.headers.get('Destination-Account')
account_name = check_account_format(req, account_name)
container_name, object_name = check_destination_header(req)
req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
api_version, account_name, container_name, object_name)
container_info = get_container_info(
req.environ, self.app)
if not container_info:
return self.app
container_info = get_container_info(
req.environ, self.app)
# To maintain backwards compatibility, container version
# location could be stored as sysmeta or not, need to check both.
@ -530,7 +514,7 @@ class VersionedWritesMiddleware(object):
if is_enabled and versions_cont:
versions_cont = unquote(versions_cont).split('/')[0]
vw_ctx = VersionedWritesContext(self.app, self.logger)
if req.method in ('PUT', 'COPY'):
if req.method == 'PUT':
resp = vw_ctx.handle_obj_versions_put(
req, versions_cont, api_version, account_name,
object_name)
@ -545,10 +529,7 @@ class VersionedWritesMiddleware(object):
return self.app
def __call__(self, env, start_response):
# making a duplicate, because if this is a COPY request, we will
# modify the PATH_INFO to find out if the 'Destination' is in a
# versioned container
req = Request(env.copy())
req = Request(env)
try:
(api_version, account, container, obj) = req.split_path(3, 4, True)
except ValueError:
@ -576,7 +557,8 @@ class VersionedWritesMiddleware(object):
allow_versioned_writes)
except HTTPException as error_response:
return error_response(env, start_response)
elif obj and req.method in ('PUT', 'COPY', 'DELETE'):
elif (obj and req.method in ('PUT', 'DELETE') and
not req.environ.get('swift.post_as_copy')):
try:
return self.object_request(
req, api_version, account, container, obj,

5
swift/common/swob.py
View File

@ -888,6 +888,11 @@ class Request(object):
return self._params_cache
str_params = params
@params.setter
def params(self, param_pairs):
self._params_cache = None
self.query_string = urllib.parse.urlencode(param_pairs)
@property
def timestamp(self):
"""

2
swift/common/wsgi.py
View File

@ -1100,7 +1100,7 @@ def make_env(env, method=None, path=None, agent='Swift', query_string=None,
'SERVER_PROTOCOL', 'swift.cache', 'swift.source',
'swift.trans_id', 'swift.authorize_override',
'swift.authorize', 'HTTP_X_USER_ID', 'HTTP_X_PROJECT_ID',
'HTTP_REFERER'):
'HTTP_REFERER', 'swift.orig_req_method', 'swift.log_info'):
if name in env:
newenv[name] = env[name]
if method:

317
swift/proxy/controllers/obj.py
View File

@ -25,7 +25,7 @@
# collected. We've seen objects hang around forever otherwise.
import six
from six.moves.urllib.parse import unquote, quote
from six.moves.urllib.parse import unquote
import collections
import itertools
@ -49,9 +49,7 @@ from swift.common.utils import (
document_iters_to_http_response_body, parse_content_range,
quorum_size, reiterate, close_if_possible)
from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_metadata, check_object_creation, \
check_copy_from_header, check_destination_header, \
check_account_format
from swift.common.constraints import check_metadata, check_object_creation
from swift.common import constraints
from swift.common.exceptions import ChunkReadTimeout, \
ChunkWriteTimeout, ConnectionTimeout, ResponseTimeout, \
@ -60,33 +58,19 @@ from swift.common.exceptions import ChunkReadTimeout, \
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.http import (
is_informational, is_success, is_client_error, is_server_error,
is_redirection, HTTP_CONTINUE, HTTP_CREATED, HTTP_MULTIPLE_CHOICES,
HTTP_INTERNAL_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE,
HTTP_INSUFFICIENT_STORAGE, HTTP_PRECONDITION_FAILED, HTTP_CONFLICT,
HTTP_UNPROCESSABLE_ENTITY, HTTP_REQUESTED_RANGE_NOT_SATISFIABLE)
is_redirection, HTTP_CONTINUE, HTTP_INTERNAL_SERVER_ERROR,
HTTP_SERVICE_UNAVAILABLE, HTTP_INSUFFICIENT_STORAGE,
HTTP_PRECONDITION_FAILED, HTTP_CONFLICT, HTTP_UNPROCESSABLE_ENTITY,
HTTP_REQUESTED_RANGE_NOT_SATISFIABLE)
from swift.common.storage_policy import (POLICIES, REPL_POLICY, EC_POLICY,
ECDriverError, PolicyError)
from swift.proxy.controllers.base import Controller, delay_denial, \
cors_validation, ResumingGetter
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \
HTTPServerError, HTTPServiceUnavailable, Request, \
HTTPClientDisconnect, HTTPUnprocessableEntity, Response, HTTPException, \
HTTPServerError, HTTPServiceUnavailable, HTTPClientDisconnect, \
HTTPUnprocessableEntity, Response, HTTPException, \
HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError
from swift.common.request_helpers import is_sys_or_user_meta, is_sys_meta, \
remove_items, copy_header_subset
def copy_headers_into(from_r, to_r):
"""
Will copy desired headers from from_r to to_r
:params from_r: a swob Request or Response
:params to_r: a swob Request or Response
"""
pass_headers = ['x-delete-at']
for k, v in from_r.headers.items():
if is_sys_or_user_meta('object', k) or k.lower() in pass_headers:
to_r.headers[k] = v
def check_content_type(req):
@ -200,8 +184,7 @@ class BaseObjectController(Controller):
self.account_name, self.container_name, self.object_name)
node_iter = self.app.iter_nodes(obj_ring, partition)
resp = self._reroute(policy)._get_or_head_response(
req, node_iter, partition, policy)
resp = self._get_or_head_response(req, node_iter, partition, policy)
if ';' in resp.headers.get('content-type', ''):
resp.content_type = clean_content_type(
@ -227,55 +210,38 @@ class BaseObjectController(Controller):
@delay_denial
def POST(self, req):
"""HTTP POST request handler."""
if self.app.object_post_as_copy:
req.method = 'PUT'
req.path_info = '/v1/%s/%s/%s' % (
self.account_name, self.container_name, self.object_name)
req.headers['Content-Length'] = 0
req.headers['X-Copy-From'] = quote('/%s/%s' % (self.container_name,
self.object_name))
req.environ['swift.post_as_copy'] = True
req.environ['swift_versioned_copy'] = True
resp = self.PUT(req)
# Older editions returned 202 Accepted on object POSTs, so we'll
# convert any 201 Created responses to that for compatibility with
# picky clients.
if resp.status_int != HTTP_CREATED:
return resp
return HTTPAccepted(request=req)
else:
error_response = check_metadata(req, 'object')
if error_response:
return error_response
container_info = self.container_info(
self.account_name, self.container_name, req)
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
if not containers:
return HTTPNotFound(request=req)
error_response = check_metadata(req, 'object')
if error_response:
return error_response
container_info = self.container_info(
self.account_name, self.container_name, req)
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
if not containers:
return HTTPNotFound(request=req)
req, delete_at_container, delete_at_part, \
delete_at_nodes = self._config_obj_expiration(req)
req, delete_at_container, delete_at_part, \
delete_at_nodes = self._config_obj_expiration(req)
# pass the policy index to storage nodes via req header
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy'])
obj_ring = self.app.get_object_ring(policy_index)
req.headers['X-Backend-Storage-Policy-Index'] = policy_index
partition, nodes = obj_ring.get_nodes(
self.account_name, self.container_name, self.object_name)
# pass the policy index to storage nodes via req header
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy'])
obj_ring = self.app.get_object_ring(policy_index)
req.headers['X-Backend-Storage-Policy-Index'] = policy_index
partition, nodes = obj_ring.get_nodes(
self.account_name, self.container_name, self.object_name)
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
headers = self._backend_requests(
req, len(nodes), container_partition, containers,
delete_at_container, delete_at_part, delete_at_nodes)
return self._post_object(req, obj_ring, partition, headers)
headers = self._backend_requests(
req, len(nodes), container_partition, containers,
delete_at_container, delete_at_part, delete_at_nodes)
return self._post_object(req, obj_ring, partition, headers)
def _backend_requests(self, req, n_outgoing,
container_partition, containers,
@ -414,133 +380,8 @@ class BaseObjectController(Controller):
return req, delete_at_container, delete_at_part, delete_at_nodes
def _handle_copy_request(self, req):
"""
This method handles copying objects based on values set in the headers
'X-Copy-From' and 'X-Copy-From-Account'
Note that if the incomming request has some conditional headers (e.g.
'Range', 'If-Match'), *source* object will be evaluated for these
headers. i.e. if PUT with both 'X-Copy-From' and 'Range', Swift will
make a partial copy as a new object.
This method was added as part of the refactoring of the PUT method and
the functionality is expected to be moved to middleware
"""
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
req.environ.setdefault('swift.log_info', []).append(
'x-copy-from:%s' % req.headers['X-Copy-From'])
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account', None)
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = check_copy_from_header(req)
source_header = '/%s/%s/%s/%s' % (
ver, src_account_name, src_container_name, src_obj_name)
source_req = req.copy_get()
# make sure the source request uses it's container_info
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
if 'swift.post_as_copy' in req.environ:
# We're COPYing one object over itself because of a POST; rely on
# the PUT for write authorization, don't require read authorization
source_req.environ['swift.authorize'] = lambda req: None
source_req.environ['swift.authorize_override'] = True
orig_obj_name = self.object_name
orig_container_name = self.container_name
orig_account_name = self.account_name
sink_req = Request.blank(req.path_info,
environ=req.environ, headers=req.headers)
self.object_name = src_obj_name
self.container_name = src_container_name
self.account_name = src_account_name
source_resp = self.GET(source_req)
# This gives middlewares a way to change the source; for example,
# this lets you COPY a SLO manifest and have the new object be the
# concatenation of the segments (like what a GET request gives
# the client), not a copy of the manifest file.
hook = req.environ.get(
'swift.copy_hook',
(lambda source_req, source_resp, sink_req: source_resp))
source_resp = hook(source_req, source_resp, sink_req)
# reset names
self.object_name = orig_obj_name
self.container_name = orig_container_name
self.account_name = orig_account_name
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
# this is a bit of ugly code, but I'm willing to live with it
# until copy request handling moves to middleware
return source_resp, None, None, None
if source_resp.content_length is None:
# This indicates a transfer-encoding: chunked source object,
# which currently only happens because there are more than
# CONTAINER_LISTING_LIMIT segments in a segmented object. In
# this case, we're going to refuse to do the server-side copy.
raise HTTPRequestEntityTooLarge(request=req)
if source_resp.content_length > constraints.MAX_FILE_SIZE:
raise HTTPRequestEntityTooLarge(request=req)
data_source = iter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
sink_req.etag = source_resp.etag
# we no longer need the X-Copy-From header
del sink_req.headers['X-Copy-From']
if 'X-Copy-From-Account' in sink_req.headers:
del sink_req.headers['X-Copy-From-Account']
if not req.content_type_manually_set:
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
fresh_meta_flag = config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false'))
if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ:
# post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# copy/update existing sysmeta and user meta
copy_headers_into(source_resp, sink_req)
copy_headers_into(req, sink_req)
# copy over x-static-large-object for POSTs and manifest copies
if 'X-Static-Large-Object' in source_resp.headers and \
(req.params.get('multipart-manifest') == 'get' or
'swift.post_as_copy' in req.environ):
sink_req.headers['X-Static-Large-Object'] = \
source_resp.headers['X-Static-Large-Object']
req = sink_req
def update_response(req, resp):
acct, path = source_resp.environ['PATH_INFO'].split('/', 3)[2:4]
resp.headers['X-Copied-From-Account'] = quote(acct)
resp.headers['X-Copied-From'] = quote(path)
if 'last-modified' in source_resp.headers:
resp.headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
copy_headers_into(req, resp)
return resp
# this is a bit of ugly code, but I'm willing to live with it
# until copy request handling moves to middleware
return None, req, data_source, update_response
def _update_content_type(self, req):
# Sometimes the 'content-type' header exists, but is set to None.
req.content_type_manually_set = True
detect_content_type = \
config_true_value(req.headers.get('x-detect-content-type'))
if detect_content_type or not req.headers.get('content-type'):
@ -549,8 +390,6 @@ class BaseObjectController(Controller):
'application/octet-stream'
if detect_content_type:
req.headers.pop('x-detect-content-type')
else:
req.content_type_manually_set = False
def _update_x_timestamp(self, req):
# Used by container sync feature
@ -744,22 +583,13 @@ class BaseObjectController(Controller):
self._update_x_timestamp(req)
# check if request is a COPY of an existing object
source_header = req.headers.get('X-Copy-From')
if source_header:
error_response, req, data_source, update_response = \
self._handle_copy_request(req)
if error_response:
return error_response
else:
def reader():
try:
return req.environ['wsgi.input'].read(
self.app.client_chunk_size)
except (ValueError, IOError) as e:
raise ChunkReadError(str(e))
data_source = iter(reader, '')
update_response = lambda req, resp: resp
def reader():
try:
return req.environ['wsgi.input'].read(
self.app.client_chunk_size)
except (ValueError, IOError) as e:
raise ChunkReadError(str(e))
data_source = iter(reader, '')
# check if object is set to be automatically deleted (i.e. expired)
req, delete_at_container, delete_at_part, \
@ -773,7 +603,7 @@ class BaseObjectController(Controller):
# send object to storage nodes
resp = self._store_object(
req, data_source, nodes, partition, outgoing_headers)
return update_response(req, resp)
return resp
@public
@cors_validation
@ -817,63 +647,6 @@ class BaseObjectController(Controller):
req, len(nodes), container_partition, containers)
return self._delete_object(req, obj_ring, partition, headers)
def _reroute(self, policy):
"""
For COPY requests we need to make sure the controller instance the
request is routed through is the correct type for the policy.
"""
if not policy:
raise HTTPServiceUnavailable('Unknown Storage Policy')
if policy.policy_type != self.policy_type:
controller = self.app.obj_controller_router[policy](
self.app, self.account_name, self.container_name,
self.object_name)
else:
controller = self
return controller
@public
@cors_validation
@delay_denial
def COPY(self, req):
"""HTTP COPY request handler."""
if not req.headers.get('Destination'):
return HTTPPreconditionFailed(request=req,
body='Destination header required')
dest_account = self.account_name
if 'Destination-Account' in req.headers:
dest_account = req.headers.get('Destination-Account')
dest_account = check_account_format(req, dest_account)
req.headers['X-Copy-From-Account'] = self.account_name
self.account_name = dest_account
del req.headers['Destination-Account']
dest_container, dest_object = check_destination_header(req)
source = '/%s/%s' % (self.container_name, self.object_name)
self.container_name = dest_container
self.object_name = dest_object
# re-write the existing request as a PUT instead of creating a new one
# since this one is already attached to the posthooklogger
# TODO: Swift now has proxy-logging middleware instead of
# posthooklogger used in before. i.e. we don't have to
# keep the code depends on evnetlet.posthooks sequence, IMHO.
# However, creating a new sub request might
# cause the possibility to hide some bugs behindes the request
# so that we should discuss whichi is suitable (new-sub-request
# vs re-write-existing-request) for Swift. [kota_]
req.method = 'PUT'
req.path_info = '/v1/%s/%s/%s' % \
(dest_account, dest_container, dest_object)
req.headers['Content-Length'] = 0
req.headers['X-Copy-From'] = quote(source)
del req.headers['Destination']
container_info = self.container_info(
dest_account, dest_container, req)
dest_policy = POLICIES.get_by_index(container_info['storage_policy'])
return self._reroute(dest_policy).PUT(req)
@ObjectControllerRouter.register(REPL_POLICY)
class ReplicatedObjectController(BaseObjectController):

15
swift/proxy/server.py
View File

@ -64,10 +64,14 @@ required_filters = [
if pipe.startswith('catch_errors')
else [])},
{'name': 'dlo', 'after_fn': lambda _junk: [
'staticweb', 'tempauth', 'keystoneauth',
'copy', 'staticweb', 'tempauth', 'keystoneauth',
'catch_errors', 'gatekeeper', 'proxy_logging']},
{'name': 'versioned_writes', 'after_fn': lambda _junk: [
'slo', 'dlo', 'staticweb', 'tempauth', 'keystoneauth',
'slo', 'dlo', 'copy', 'staticweb', 'tempauth',
'keystoneauth', 'catch_errors', 'gatekeeper', 'proxy_logging']},
# Put copy before dlo, slo and versioned_writes
{'name': 'copy', 'after_fn': lambda _junk: [
'staticweb', 'tempauth', 'keystoneauth',
'catch_errors', 'gatekeeper', 'proxy_logging']}]
@ -107,8 +111,6 @@ class Application(object):
int(conf.get('recheck_account_existence', 60))
self.allow_account_management = \
config_true_value(conf.get('allow_account_management', 'no'))
self.object_post_as_copy = \
config_true_value(conf.get('object_post_as_copy', 'true'))
self.container_ring = container_ring or Ring(swift_dir,
ring_name='container')
self.account_ring = account_ring or Ring(swift_dir,
@ -392,8 +394,7 @@ class Application(object):
# controller's method indicates it'd like to gather more
# information and try again later.
resp = req.environ['swift.authorize'](req)
if not resp and not req.headers.get('X-Copy-From-Account') \
and not req.headers.get('Destination-Account'):
if not resp:
# No resp means authorized, no delayed recheck required.
old_authorize = req.environ['swift.authorize']
else:
@ -404,7 +405,7 @@ class Application(object):
# Save off original request method (GET, POST, etc.) in case it
# gets mutated during handling. This way logging can display the
# method the client actually sent.
req.environ['swift.orig_req_method'] = req.method
req.environ.setdefault('swift.orig_req_method', req.method)
try:
if old_authorize:
req.environ.pop('swift.authorize', None)

94
test/functional/tests.py
View File

@ -1306,12 +1306,10 @@ class TestFile(Base):
acct,
'%s%s' % (prefix, self.env.container),
Utils.create_name()))
if acct == acct2:
# there is no such source container
# and foreign user can have no permission to read it
self.assert_status(403)
else:
self.assert_status(404)
# there is no such source container but user has
# permissions to do a GET (done internally via COPY) for
# objects in his own account.
self.assert_status(404)
self.assertFalse(file_item.copy_account(
acct,
@ -1325,12 +1323,10 @@ class TestFile(Base):
acct,
'%s%s' % (prefix, self.env.container),
Utils.create_name()))
if acct == acct2:
# there is no such object
# and foreign user can have no permission to read it
self.assert_status(403)
else:
self.assert_status(404)
# there is no such source container but user has
# permissions to do a GET (done internally via COPY) for
# objects in his own account.
self.assert_status(404)
self.assertFalse(file_item.copy_account(
acct,
@ -2677,6 +2673,23 @@ class TestFileComparisonUTF8(Base2, TestFileComparison):
class TestSloEnv(object):
slo_enabled = None # tri-state: None initially, then True/False
@classmethod
def create_segments(cls, container):
seg_info = {}
for letter, size in (('a', 1024 * 1024),
('b', 1024 * 1024),
('c', 1024 * 1024),
('d', 1024 * 1024),
('e', 1)):
seg_name = "seg_%s" % letter
file_item = container.file(seg_name)
file_item.write(letter * size)
seg_info[seg_name] = {
'size_bytes': size,
'etag': file_item.md5,
'path': '/%s/%s' % (container.name, seg_name)}
return seg_info
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
@ -2711,19 +2724,7 @@ class TestSloEnv(object):
if not cont.create():
raise ResponseError(cls.conn.response)
cls.seg_info = seg_info = {}
for letter, size in (('a', 1024 * 1024),
('b', 1024 * 1024),
('c', 1024 * 1024),
('d', 1024 * 1024),
('e', 1)):
seg_name = "seg_%s" % letter
file_item = cls.container.file(seg_name)
file_item.write(letter * size)
seg_info[seg_name] = {
'size_bytes': size,
'etag': file_item.md5,
'path': '/%s/%s' % (cls.container.name, seg_name)}
cls.seg_info = seg_info = cls.create_segments(cls.container)
file_item = cls.container.file("manifest-abcde")
file_item.write(
@ -3125,8 +3126,9 @@ class TestSlo(Base):
def test_slo_copy_the_manifest(self):
file_item = self.env.container.file("manifest-abcde")
file_item.copy(self.env.container.name, "copied-abcde-manifest-only",
parms={'multipart-manifest': 'get'})
self.assertTrue(file_item.copy(self.env.container.name,
"copied-abcde-manifest-only",
parms={'multipart-manifest': 'get'}))
copied = self.env.container.file("copied-abcde-manifest-only")
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
@ -3157,10 +3159,40 @@ class TestSlo(Base):
self.assertTrue(dest_cont.create(hdrs={
'X-Container-Write': self.env.conn.user_acl
}))
file_item.copy_account(acct,
dest_cont,
"copied-abcde-manifest-only",
parms={'multipart-manifest': 'get'})
# manifest copy will fail because there is no read access to segments
# in destination account
file_item.copy_account(
acct, dest_cont, "copied-abcde-manifest-only",
parms={'multipart-manifest': 'get'})
self.assertEqual(400, file_item.conn.response.status)
resp_body = file_item.conn.response.read()
self.assertEqual(5, resp_body.count('403 Forbidden'),
'Unexpected response body %r' % resp_body)
# create segments container in account2 with read access for account1
segs_container = self.env.account2.container(self.env.container.name)
self.assertTrue(segs_container.create(hdrs={
'X-Container-Read': self.env.conn.user_acl
}))
# manifest copy will still fail because there are no segments in
# destination account
file_item.copy_account(
acct, dest_cont, "copied-abcde-manifest-only",
parms={'multipart-manifest': 'get'})
self.assertEqual(400, file_item.conn.response.status)
resp_body = file_item.conn.response.read()
self.assertEqual(5, resp_body.count('404 Not Found'),
'Unexpected response body %r' % resp_body)
# create segments in account2 container with same name as in account1,
# manifest copy now succeeds
self.env.create_segments(segs_container)
self.assertTrue(file_item.copy_account(
acct, dest_cont, "copied-abcde-manifest-only",
parms={'multipart-manifest': 'get'}))
copied = dest_cont.file("copied-abcde-manifest-only")
copied_contents = copied.read(parms={'multipart-manifest': 'get'})

6
test/unit/common/middleware/helpers.py
View File

@ -20,6 +20,7 @@ from copy import deepcopy
from hashlib import md5
from swift.common import swob
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.swob import HTTPNotImplemented
from swift.common.utils import split_path
from test.unit import FakeLogger, FakeRing
@ -43,6 +44,8 @@ class FakeSwift(object):
"""
A good-enough fake Swift proxy server to use in testing middleware.
"""
ALLOWED_METHODS = [
'PUT', 'POST', 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'REPLICATE']
def __init__(self):
self._calls = []
@ -71,6 +74,9 @@ class FakeSwift(object):
def __call__(self, env, start_response):
method = env['REQUEST_METHOD']
if method not in self.ALLOWED_METHODS:
raise HTTPNotImplemented()
path = env['PATH_INFO']
_, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4,
rest_with_last=True)

169
test/unit/common/middleware/test_account_quotas.py
View File

@ -13,9 +13,10 @@
import unittest
from swift.common.swob import Request, wsgify, HTTPForbidden
from swift.common.swob import Request, wsgify, HTTPForbidden, \
HTTPException
from swift.common.middleware import account_quotas
from swift.common.middleware import account_quotas, copy
from swift.proxy.controllers.base import _get_cache_key, \
headers_to_account_info, get_object_env_key, \
@ -245,84 +246,6 @@ class TestAccountQuota(unittest.TestCase):
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_exceed_bytes_quota_copy_from(self):
headers = [('x-account-bytes-used', '500'),
('x-account-meta-quota-bytes', '1000'),
('content-length', '1000')]
app = account_quotas.AccountQuotaMiddleware(FakeApp(headers))
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(app)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_bytes_quota_copy_verb(self):
headers = [('x-account-bytes-used', '500'),
('x-account-meta-quota-bytes', '1000'),
('content-length', '1000')]
app = account_quotas.AccountQuotaMiddleware(FakeApp(headers))
cache = FakeCache(None)
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(app)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_bytes_quota_copy_from(self):
headers = [('x-account-bytes-used', '0'),
('x-account-meta-quota-bytes', '1000'),
('content-length', '1000')]
app = account_quotas.AccountQuotaMiddleware(FakeApp(headers))
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_not_exceed_bytes_quota_copy_verb(self):
headers = [('x-account-bytes-used', '0'),
('x-account-meta-quota-bytes', '1000'),
('content-length', '1000')]
app = account_quotas.AccountQuotaMiddleware(FakeApp(headers))
cache = FakeCache(None)
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_quota_copy_from_no_src(self):
headers = [('x-account-bytes-used', '0'),
('x-account-meta-quota-bytes', '1000')]
app = account_quotas.AccountQuotaMiddleware(FakeApp(headers))
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o3'})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_quota_copy_from_bad_src(self):
headers = [('x-account-bytes-used', '0'),
('x-account-meta-quota-bytes', '1000')]
app = account_quotas.AccountQuotaMiddleware(FakeApp(headers))
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': 'bad_path'})
res = req.get_response(app)
self.assertEqual(res.status_int, 412)
def test_exceed_bytes_quota_reseller(self):
headers = [('x-account-bytes-used', '1000'),
('x-account-meta-quota-bytes', '0')]
@ -485,5 +408,91 @@ class TestAccountQuota(unittest.TestCase):
self.assertEqual(res.status_int, 200)
class AccountQuotaCopyingTestCases(unittest.TestCase):
def setUp(self):
self.app = FakeApp()
self.aq_filter = account_quotas.filter_factory({})(self.app)
self.copy_filter = copy.filter_factory({})(self.aq_filter)
def test_exceed_bytes_quota_copy_from(self):
headers = [('x-account-bytes-used', '500'),
('x-account-meta-quota-bytes', '1000'),
('content-length', '1000')]
self.app.headers = headers
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_bytes_quota_copy_verb(self):
headers = [('x-account-bytes-used', '500'),
('x-account-meta-quota-bytes', '1000'),
('content-length', '1000')]
self.app.headers = headers
cache = FakeCache(None)
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_bytes_quota_copy_from(self):
headers = [('x-account-bytes-used', '0'),
('x-account-meta-quota-bytes', '1000'),
('content-length', '1000')]
self.app.headers = headers
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_not_exceed_bytes_quota_copy_verb(self):
headers = [('x-account-bytes-used', '0'),
('x-account-meta-quota-bytes', '1000'),
('content-length', '1000')]
self.app.headers = headers
cache = FakeCache(None)
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_quota_copy_from_no_src(self):
headers = [('x-account-bytes-used', '0'),
('x-account-meta-quota-bytes', '1000')]
self.app.headers = headers
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o3'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_quota_copy_from_bad_src(self):
headers = [('x-account-bytes-used', '0'),
('x-account-meta-quota-bytes', '1000')]
self.app.headers = headers
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': 'bad_path'})
with self.assertRaises(HTTPException) as catcher:
req.get_response(self.copy_filter)
self.assertEqual(412, catcher.exception.status_int)
if __name__ == '__main__':
unittest.main()

1183
test/unit/common/middleware/test_copy.py Normal file
View File

File diff suppressed because it is too large Load Diff

101
test/unit/common/middleware/test_dlo.py
View File

@ -803,107 +803,6 @@ class TestDloGetManifest(DloTestCase):
self.assertTrue(auth_got_called[0] > 1)
def fake_start_response(*args, **kwargs):
pass
class TestDloCopyHook(DloTestCase):
def setUp(self):
super(TestDloCopyHook, self).setUp()
self.app.register(
'GET', '/v1/AUTH_test/c/o1', swob.HTTPOk,
{'Content-Length': '10', 'Etag': 'o1-etag'},
"aaaaaaaaaa")
self.app.register(
'GET', '/v1/AUTH_test/c/o2', swob.HTTPOk,
{'Content-Length': '10', 'Etag': 'o2-etag'},
"bbbbbbbbbb")
self.app.register(
'GET', '/v1/AUTH_test/c/man',
swob.HTTPOk, {'X-Object-Manifest': 'c/o'},
"manifest-contents")
lm = '2013-11-22T02:42:13.781760'
ct = 'application/octet-stream'
segs = [{"hash": "o1-etag", "bytes": 10, "name": "o1",
"last_modified": lm, "content_type": ct},
{"hash": "o2-etag", "bytes": 5, "name": "o2",
"last_modified": lm, "content_type": ct}]
self.app.register(
'GET', '/v1/AUTH_test/c?format=json&prefix=o',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs))
copy_hook = [None]
# slip this guy in there to pull out the hook
def extract_copy_hook(env, sr):
copy_hook[0] = env.get('swift.copy_hook')
return self.app(env, sr)
self.dlo = dlo.filter_factory({})(extract_copy_hook)
req = swob.Request.blank('/v1/AUTH_test/c/o1',
environ={'REQUEST_METHOD': 'GET'})
self.dlo(req.environ, fake_start_response)
self.copy_hook = copy_hook[0]
self.assertTrue(self.copy_hook is not None) # sanity check
def test_copy_hook_passthrough(self):
source_req = swob.Request.blank(
'/v1/AUTH_test/c/man',
environ={'REQUEST_METHOD': 'GET'})
sink_req = swob.Request.blank(
'/v1/AUTH_test/c/man',
environ={'REQUEST_METHOD': 'PUT'})
source_resp = swob.Response(request=source_req, status=200)
# no X-Object-Manifest header, so do nothing
modified_resp = self.copy_hook(source_req, source_resp, sink_req)
self.assertTrue(modified_resp is source_resp)
def test_copy_hook_manifest(self):
source_req = swob.Request.blank(
'/v1/AUTH_test/c/man',
environ={'REQUEST_METHOD': 'GET'})
sink_req = swob.Request.blank(
'/v1/AUTH_test/c/man',
environ={'REQUEST_METHOD': 'PUT'})
source_resp = swob.Response(
request=source_req, status=200,
headers={"X-Object-Manifest": "c/o"},
app_iter=["manifest"])
# it's a manifest, so copy the segments to make a normal object
modified_resp = self.copy_hook(source_req, source_resp, sink_req)
self.assertTrue(modified_resp is not source_resp)
self.assertEqual(modified_resp.etag,
hashlib.md5("o1-etago2-etag").hexdigest())
self.assertEqual(sink_req.headers.get('X-Object-Manifest'), None)
def test_copy_hook_manifest_with_multipart_manifest_get(self):
source_req = swob.Request.blank(
'/v1/AUTH_test/c/man',
environ={'REQUEST_METHOD': 'GET',
'QUERY_STRING': 'multipart-manifest=get'})
sink_req = swob.Request.blank(
'/v1/AUTH_test/c/man',
environ={'REQUEST_METHOD': 'PUT'})
source_resp = swob.Response(
request=source_req, status=200,
headers={"X-Object-Manifest": "c/o"},
app_iter=["manifest"])
# make sure the sink request (the backend PUT) gets X-Object-Manifest
# on it, but that's all
modified_resp = self.copy_hook(source_req, source_resp, sink_req)
self.assertTrue(modified_resp is source_resp)
self.assertEqual(sink_req.headers.get('X-Object-Manifest'), 'c/o')
class TestDloConfiguration(unittest.TestCase):
"""
For backwards compatibility, we will read a couple of values out of the

323
test/unit/common/middleware/test_quotas.py
View File

@ -15,8 +15,9 @@
import unittest
from swift.common.swob import Request, HTTPUnauthorized
from swift.common.middleware import container_quotas
from swift.common.swob import Request, HTTPUnauthorized, HTTPOk, HTTPException
from swift.common.middleware import container_quotas, copy
from test.unit.common.middleware.helpers import FakeSwift
class FakeCache(object):
@ -95,32 +96,6 @@ class TestContainerQuotas(unittest.TestCase):
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_bytes_quota_copy_from(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.object/a/c2/o2': {'length': 10},
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(app)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_bytes_quota_copy_verb(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.object/a/c2/o2': {'length': 10},
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(app)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_bytes_quota(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
@ -131,60 +106,6 @@ class TestContainerQuotas(unittest.TestCase):
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_not_exceed_bytes_quota_copy_from(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.object/a/c2/o2': {'length': 10},
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_not_exceed_bytes_quota_copy_verb(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.object/a/c2/o2': {'length': 10},
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_bytes_quota_copy_from_no_src(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.object/a/c2/o2': {'length': 10},
'swift.cache': cache},
headers={'x-copy-from': '/c2/o3'})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_bytes_quota_copy_from_bad_src(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': 'bad_path'})
res = req.get_response(app)
self.assertEqual(res.status_int, 412)
def test_bytes_quota_copy_verb_no_src(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c2/o3',
environ={'REQUEST_METHOD': 'COPY',
'swift.object/a/c2/o2': {'length': 10},
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_exceed_counts_quota(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
@ -196,61 +117,6 @@ class TestContainerQuotas(unittest.TestCase):
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_counts_quota_copy_from(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.object/a/c2/o2': {'length': 10},
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(app)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_counts_quota_copy_verb(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(app)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_counts_quota_copy_cross_account_verb(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'},
'status': 200, 'object_count': 1}
a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'},
'status': 200, 'object_count': 1}
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.container/a/c': a_c_cache,
'swift.container/a2/c': a2_c_cache},
headers={'Destination': '/c/o',
'Destination-Account': 'a2'})
res = req.get_response(app)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_counts_quota_copy_cross_account_PUT_verb(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'},
'status': 200, 'object_count': 1}
a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'},
'status': 200, 'object_count': 1}
req = Request.blank('/v1/a2/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.container/a/c': a_c_cache,
'swift.container/a2/c': a2_c_cache},
headers={'X-Copy-From': '/c2/o2',
'X-Copy-From-Account': 'a'})
res = req.get_response(app)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_counts_quota(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
@ -261,26 +127,6 @@ class TestContainerQuotas(unittest.TestCase):
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_not_exceed_counts_quota_copy_from(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_not_exceed_counts_quota_copy_verb(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_invalid_quotas(self):
req = Request.blank(
'/v1/a/c',
@ -346,5 +192,168 @@ class TestContainerQuotas(unittest.TestCase):
res = req.get_response(app)
self.assertEqual(res.status_int, 401)
class ContainerQuotaCopyingTestCases(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.cq_filter = container_quotas.filter_factory({})(self.app)
self.copy_filter = copy.filter_factory({})(self.cq_filter)
def test_exceed_bytes_quota_copy_verb(self):
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_bytes_quota_copy_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register(
'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_exceed_counts_quota_copy_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_counts_quota_copy_cross_account_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'},
'status': 200, 'object_count': 1}
a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'},
'status': 200, 'object_count': 1}
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.container/a/c': a_c_cache,
'swift.container/a2/c': a2_c_cache},
headers={'Destination': '/c/o',
'Destination-Account': 'a2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_counts_quota_copy_cross_account_PUT_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {}, 'passed')
a_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '2'},
'status': 200, 'object_count': 1}
a2_c_cache = {'storage_policy': '0', 'meta': {'quota-count': '1'},
'status': 200, 'object_count': 1}
req = Request.blank('/v1/a2/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.container/a/c': a_c_cache,
'swift.container/a2/c': a2_c_cache},
headers={'X-Copy-From': '/c2/o2',
'X-Copy-From-Account': 'a'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_exceed_bytes_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_bytes_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register(
'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_bytes_quota_copy_from_no_src(self):
self.app.register('GET', '/v1/a/c2/o3', HTTPOk, {}, 'passed')
self.app.register(
'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o3'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_bytes_quota_copy_from_bad_src(self):
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': 'bad_path'})
with self.assertRaises(HTTPException) as catcher:
req.get_response(self.copy_filter)
self.assertEqual(412, catcher.exception.status_int)
def test_exceed_counts_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, 'Upload exceeds quota.')
def test_not_exceed_counts_quota_copy_from(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register(
'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_not_exceed_counts_quota_copy_verb(self):
self.app.register('GET', '/v1/a/c2/o2', HTTPOk,
{'Content-Length': '10'}, 'passed')
self.app.register(
'PUT', '/v1/a/c/o', HTTPOk, {}, 'passed')
cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
if __name__ == '__main__':
unittest.main()

66
test/unit/common/middleware/test_slo.py
View File

@ -26,7 +26,7 @@ from swift.common import swob, utils
from swift.common.exceptions import ListingIterError, SegmentError
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.middleware import slo
from swift.common.swob import Request, Response, HTTPException
from swift.common.swob import Request, HTTPException
from swift.common.utils import quote, closing_if_possible, close_if_possible
from test.unit.common.middleware.helpers import FakeSwift
@ -2653,70 +2653,6 @@ class TestSloBulkLogger(unittest.TestCase):
self.assertTrue(slo_mware.logger is slo_mware.bulk_deleter.logger)
class TestSloCopyHook(SloTestCase):
def setUp(self):
super(TestSloCopyHook, self).setUp()
self.app.register(
'GET', '/v1/AUTH_test/c/o', swob.HTTPOk,
{'Content-Length': '3', 'Etag': md5hex("obj")}, "obj")
self.app.register(
'GET', '/v1/AUTH_test/c/man',
swob.HTTPOk, {'Content-Type': 'application/json',
'X-Static-Large-Object': 'true'},
json.dumps([{'name': '/c/o', 'hash': md5hex("obj"),
'bytes': '3'}]))
self.app.register(
'COPY', '/v1/AUTH_test/c/o', swob.HTTPCreated, {})
copy_hook = [None]
# slip this guy in there to pull out the hook
def extract_copy_hook(env, sr):
if env['REQUEST_METHOD'] == 'COPY':
copy_hook[0] = env['swift.copy_hook']
return self.app(env, sr)
self.slo = slo.filter_factory({})(extract_copy_hook)
req = Request.blank('/v1/AUTH_test/c/o',
environ={'REQUEST_METHOD': 'COPY'})
self.slo(req.environ, fake_start_response)
self.copy_hook = copy_hook[0]
self.assertTrue(self.copy_hook is not None) # sanity check
def test_copy_hook_passthrough(self):
source_req = Request.blank(
'/v1/AUTH_test/c/o',
environ={'REQUEST_METHOD': 'GET'})
sink_req = Request.blank(
'/v1/AUTH_test/c/o',
environ={'REQUEST_METHOD': 'PUT'})
# no X-Static-Large-Object header, so do nothing
source_resp = Response(request=source_req, status=200)
modified_resp = self.copy_hook(source_req, source_resp, sink_req)
self.assertTrue(modified_resp is source_resp)
def test_copy_hook_manifest(self):
source_req = Request.blank(
'/v1/AUTH_test/c/o',
environ={'REQUEST_METHOD': 'GET'})
sink_req = Request.blank(
'/v1/AUTH_test/c/o',
environ={'REQUEST_METHOD': 'PUT'})
source_resp = Response(request=source_req, status=200,
headers={"X-Static-Large-Object": "true"},
app_iter=[json.dumps([{'name': '/c/o',
'hash': md5hex("obj"),
'bytes': '3'}])])
modified_resp = self.copy_hook(source_req, source_resp, sink_req)
self.assertTrue(modified_resp is not source_resp)
self.assertEqual(modified_resp.etag, md5hex(md5hex("obj")))
class TestSwiftInfo(unittest.TestCase):
def setUp(self):
utils._swift_info = {}

225
test/unit/common/middleware/test_versioned_writes.py
View File

@ -19,7 +19,7 @@ import os
import time
import unittest
from swift.common import swob
from swift.common.middleware import versioned_writes
from swift.common.middleware import versioned_writes, copy
from swift.common.swob import Request
from test.unit.common.middleware.helpers import FakeSwift
@ -259,6 +259,23 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase):
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_put_object_post_as_copy(self):
# PUTs due to a post-as-copy should NOT cause a versioning op
self.app.register(
'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed')
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100',
'swift.post_as_copy': True})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
self.assertEqual(1, self.app.call_count)
def test_put_first_object_success(self):
self.app.register(
'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
@ -333,7 +350,7 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase):
def test_delete_object_no_versioning_with_container_config_true(self):
# set False to versions_write obviously and expect no GET versioning
# container and PUT called (just delete object as normal)
# container and GET/PUT called (just delete object as normal)
self.vw.conf = {'allow_versioned_writes': 'false'}
self.app.register(
'DELETE', '/v1/a/c/o', swob.HTTPNoContent, {}, 'passed')
@ -351,25 +368,6 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase):
self.assertTrue('GET' not in called_method)
self.assertEqual(1, self.app.call_count)
def test_copy_object_no_versioning_with_container_config_true(self):
# set False to versions_write obviously and expect no extra
# COPY called (just copy object as normal)
self.vw.conf = {'allow_versioned_writes': 'false'}
self.app.register(
'COPY', '/v1/a/c/o', swob.HTTPCreated, {}, None)
cache = FakeCache({'versions': 'ver_cont'})
req = Request.blank(
'/v1/a/c/o',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
called_method = \
[method for (method, path, rheaders) in self.app._calls]
self.assertTrue('COPY' in called_method)
self.assertEqual(called_method.count('COPY'), 1)
def test_new_version_success(self):
self.app.register(
'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed')
@ -476,77 +474,6 @@ class VersionedWritesTestCase(VersionedWritesBaseTestCase):
self.assertEqual('PUT', method)
self.assertEqual('/v1/a/ver_cont/001o/0000000000.00000', path)
def test_copy_first_version(self):
self.app.register(
'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
self.assertEqual(2, self.app.call_count)
def test_copy_new_version(self):
self.app.register(
'COPY', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed')
self.app.register(
'PUT', '/v1/a/ver_cont/007tgt_obj/0000000001.00000', swob.HTTPOk,
{}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
self.assertEqual(3, self.app.call_count)
def test_copy_new_version_different_account(self):
self.app.register(
'COPY', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed')
self.app.register(
'PUT', '/v1/tgt_a/ver_cont/007tgt_obj/0000000001.00000',
swob.HTTPOk, {}, None)
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/src_a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj',
'Destination-Account': 'tgt_a'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '200 OK')
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
self.assertEqual(3, self.app.call_count)
def test_copy_new_version_bogus_account(self):
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/src_a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj',
'Destination-Account': '/im/on/a/boat'})
status, headers, body = self.call_vw(req)
self.assertEqual(status, '412 Precondition Failed')
def test_delete_first_object_success(self):
self.app.register(
'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
@ -1057,3 +984,117 @@ class VersionedWritesOldContainersTestCase(VersionedWritesBaseTestCase):
('PUT', '/v1/a/c/o'),
('DELETE', '/v1/a/ver_cont/001o/2'),
])
class VersionedWritesCopyingTestCase(VersionedWritesBaseTestCase):
# verify interaction of copy and versioned_writes middlewares
def setUp(self):
self.app = FakeSwift()
conf = {'allow_versioned_writes': 'true'}
self.vw = versioned_writes.filter_factory(conf)(self.app)
self.filter = copy.filter_factory({})(self.vw)
def call_filter(self, req, **kwargs):
return self.call_app(req, app=self.filter, **kwargs)
def test_copy_first_version(self):
# no existing object to move to the versions container
self.app.register(
'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound, {}, None)
self.app.register(
'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed')
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
# note the GET on tgt_cont/tgt_obj is pre-authed
self.assertEqual(3, self.app.call_count, self.app.calls)
def test_copy_new_version(self):
# existing object should be moved to versions container
self.app.register(
'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed')
self.app.register(
'PUT', '/v1/a/ver_cont/007tgt_obj/0000000001.00000', swob.HTTPOk,
{}, None)
self.app.register(
'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed')
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj'})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
self.assertEqual(4, self.app.call_count)
def test_copy_new_version_different_account(self):
self.app.register(
'GET', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'GET', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk,
{'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed')
self.app.register(
'PUT', '/v1/tgt_a/ver_cont/007tgt_obj/0000000001.00000',
swob.HTTPOk, {}, None)
self.app.register(
'PUT', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPCreated, {},
'passed')
cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
req = Request.blank(
'/v1/src_a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
'CONTENT_LENGTH': '100'},
headers={'Destination': 'tgt_cont/tgt_obj',
'Destination-Account': 'tgt_a'})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/src_a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/tgt_a/tgt_cont/tgt_obj', self.authorized[1].path)
self.assertEqual(4, self.app.call_count)
def test_copy_object_no_versioning_with_container_config_true(self):
# set False to versions_write obviously and expect no extra
# COPY called (just copy object as normal)
self.vw.conf = {'allow_versioned_writes': 'false'}
self.app.register(
'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
self.app.register(
'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed')
cache = FakeCache({'versions': 'ver_cont'})
req = Request.blank(
'/v1/a/src_cont/src_obj',
environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache},
headers={'Destination': '/tgt_cont/tgt_obj'})
status, headers, body = self.call_filter(req)
self.assertEqual(status, '201 Created')
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
self.assertEqual(2, self.app.call_count)

81
test/unit/common/test_constraints.py
View File

@ -173,33 +173,6 @@ class TestConstraints(unittest.TestCase):
'/', headers=headers), 'object_name').status_int,
HTTP_NOT_IMPLEMENTED)
def test_check_object_creation_copy(self):
headers = {'Content-Length': '0',
'X-Copy-From': 'c/o2',
'Content-Type': 'text/plain'}
self.assertEqual(constraints.check_object_creation(Request.blank(
'/', headers=headers), 'object_name'), None)
headers = {'Content-Length': '1',
'X-Copy-From': 'c/o2',
'Content-Type': 'text/plain'}
self.assertEqual(constraints.check_object_creation(Request.blank(
'/', headers=headers), 'object_name').status_int,
HTTP_BAD_REQUEST)
headers = {'Transfer-Encoding': 'chunked',
'X-Copy-From': 'c/o2',
'Content-Type': 'text/plain'}
self.assertEqual(constraints.check_object_creation(Request.blank(
'/', headers=headers), 'object_name'), None)
# a content-length header is always required
headers = {'X-Copy-From': 'c/o2',
'Content-Type': 'text/plain'}
self.assertEqual(constraints.check_object_creation(Request.blank(
'/', headers=headers), 'object_name').status_int,
HTTP_LENGTH_REQUIRED)
def test_check_object_creation_name_length(self):
headers = {'Transfer-Encoding': 'chunked',
'Content-Type': 'text/plain'}
@ -459,60 +432,6 @@ class TestConstraints(unittest.TestCase):
self.assertTrue(c.MAX_HEADER_SIZE > c.MAX_META_NAME_LENGTH)
self.assertTrue(c.MAX_HEADER_SIZE > c.MAX_META_VALUE_LENGTH)
def test_validate_copy_from(self):
req = Request.blank(
'/v/a/c/o',
headers={'x-copy-from': 'c/o2'})
src_cont, src_obj = constraints.check_copy_from_header(req)
self.assertEqual(src_cont, 'c')
self.assertEqual(src_obj, 'o2')
req = Request.blank(
'/v/a/c/o',
headers={'x-copy-from': 'c/subdir/o2'})
src_cont, src_obj = constraints.check_copy_from_header(req)
self.assertEqual(src_cont, 'c')
self.assertEqual(src_obj, 'subdir/o2')
req = Request.blank(
'/v/a/c/o',
headers={'x-copy-from': '/c/o2'})
src_cont, src_obj = constraints.check_copy_from_header(req)
self.assertEqual(src_cont, 'c')
self.assertEqual(src_obj, 'o2')
def test_validate_bad_copy_from(self):
req = Request.blank(
'/v/a/c/o',
headers={'x-copy-from': 'bad_object'})
self.assertRaises(HTTPException,
constraints.check_copy_from_header, req)
def test_validate_destination(self):
req = Request.blank(
'/v/a/c/o',
headers={'destination': 'c/o2'})
src_cont, src_obj = constraints.check_destination_header(req)
self.assertEqual(src_cont, 'c')
self.assertEqual(src_obj, 'o2')
req = Request.blank(
'/v/a/c/o',
headers={'destination': 'c/subdir/o2'})
src_cont, src_obj = constraints.check_destination_header(req)
self.assertEqual(src_cont, 'c')
self.assertEqual(src_obj, 'subdir/o2')
req = Request.blank(
'/v/a/c/o',
headers={'destination': '/c/o2'})
src_cont, src_obj = constraints.check_destination_header(req)
self.assertEqual(src_cont, 'c')
self.assertEqual(src_obj, 'o2')
def test_validate_bad_destination(self):
req = Request.blank(
'/v/a/c/o',
headers={'destination': 'bad_object'})
self.assertRaises(HTTPException,
constraints.check_destination_header, req)
def test_check_account_format(self):
req = Request.blank(
'/v/a/c/o',

13
test/unit/common/test_swob.py
View File

@ -431,9 +431,10 @@ class TestRequest(unittest.TestCase):
def test_invalid_req_environ_property_args(self):
# getter only property
try:
swift.common.swob.Request.blank('/', params={'a': 'b'})
swift.common.swob.Request.blank(
'/', host_url='http://example.com:8080/v1/a/c/o')
except TypeError as e:
self.assertEqual("got unexpected keyword argument 'params'",
self.assertEqual("got unexpected keyword argument 'host_url'",
str(e))
else:
self.assertTrue(False, "invalid req_environ_property "
@ -525,6 +526,14 @@ class TestRequest(unittest.TestCase):
self.assertEqual(req.params['a'], 'b')
self.assertEqual(req.params['c'], 'd')
new_params = {'e': 'f', 'g': 'h'}
req.params = new_params
self.assertDictEqual(new_params, req.params)
new_params = (('i', 'j'), ('k', 'l'))
req.params = new_params
self.assertDictEqual(dict(new_params), req.params)
def test_timestamp_missing(self):
req = swift.common.swob.Request.blank('/')
self.assertRaises(exceptions.InvalidTimestamp,

13
test/unit/common/test_wsgi.py
View File

@ -136,6 +136,11 @@ class TestWSGI(unittest.TestCase):
expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware
self.assertTrue(isinstance(app, expected))
app = app.app
expected = \
swift.common.middleware.copy.ServerSideCopyMiddleware
self.assertIsInstance(app, expected)
app = app.app
expected = swift.common.middleware.dlo.DynamicLargeObject
self.assertTrue(isinstance(app, expected))
@ -1437,6 +1442,7 @@ class TestPipelineModification(unittest.TestCase):
self.assertEqual(self.pipeline_modules(app),
['swift.common.middleware.catch_errors',
'swift.common.middleware.gatekeeper',
'swift.common.middleware.copy',
'swift.common.middleware.dlo',
'swift.common.middleware.versioned_writes',
'swift.proxy.server'])
@ -1468,6 +1474,7 @@ class TestPipelineModification(unittest.TestCase):
self.assertEqual(self.pipeline_modules(app),
['swift.common.middleware.catch_errors',
'swift.common.middleware.gatekeeper',
'swift.common.middleware.copy',
'swift.common.middleware.dlo',
'swift.common.middleware.versioned_writes',
'swift.common.middleware.healthcheck',
@ -1506,6 +1513,7 @@ class TestPipelineModification(unittest.TestCase):
self.assertEqual(self.pipeline_modules(app),
['swift.common.middleware.catch_errors',
'swift.common.middleware.gatekeeper',
'swift.common.middleware.copy',
'swift.common.middleware.slo',
'swift.common.middleware.dlo',
'swift.common.middleware.versioned_writes',
@ -1605,6 +1613,7 @@ class TestPipelineModification(unittest.TestCase):
self.assertEqual(self.pipeline_modules(app), [
'swift.common.middleware.catch_errors',
'swift.common.middleware.gatekeeper',
'swift.common.middleware.copy',
'swift.common.middleware.dlo',
'swift.common.middleware.versioned_writes',
'swift.common.middleware.healthcheck',
@ -1619,6 +1628,7 @@ class TestPipelineModification(unittest.TestCase):
'swift.common.middleware.gatekeeper',
'swift.common.middleware.healthcheck',
'swift.common.middleware.catch_errors',
'swift.common.middleware.copy',
'swift.common.middleware.dlo',
'swift.common.middleware.versioned_writes',
'swift.proxy.server'])
@ -1632,6 +1642,7 @@ class TestPipelineModification(unittest.TestCase):
'swift.common.middleware.healthcheck',
'swift.common.middleware.catch_errors',
'swift.common.middleware.gatekeeper',
'swift.common.middleware.copy',
'swift.common.middleware.dlo',
'swift.common.middleware.versioned_writes',
'swift.proxy.server'])
@ -1666,7 +1677,7 @@ class TestPipelineModification(unittest.TestCase):
tempdir, policy.ring_name + '.ring.gz')
app = wsgi.loadapp(conf_path)
proxy_app = app.app.app.app.app.app
proxy_app = app.app.app.app.app.app.app
self.assertEqual(proxy_app.account_ring.serialized_path,
account_ring_path)
self.assertEqual(proxy_app.container_ring.serialized_path,

209
test/unit/proxy/controllers/test_obj.py
View File

@ -649,7 +649,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
def test_PUT_error_during_transfer_data(self):
class FakeReader(object):
def read(self, size):
raise exceptions.ChunkReadError('exception message')
raise IOError('error message')
req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
body='test body')
@ -747,62 +747,6 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 404)
def test_POST_as_COPY_simple(self):
req = swift.common.swob.Request.blank('/v1/a/c/o', method='POST')
get_resp = [200] * self.obj_ring.replicas + \
[404] * self.obj_ring.max_more_nodes
put_resp = [201] * self.obj_ring.replicas
codes = get_resp + put_resp
with set_http_connect(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
self.assertEqual(req.environ['QUERY_STRING'], '')
self.assertTrue('swift.post_as_copy' in req.environ)
def test_POST_as_COPY_static_large_object(self):
req = swift.common.swob.Request.blank('/v1/a/c/o', method='POST')
get_resp = [200] * self.obj_ring.replicas + \
[404] * self.obj_ring.max_more_nodes
put_resp = [201] * self.obj_ring.replicas
codes = get_resp + put_resp
slo_headers = \
[{'X-Static-Large-Object': True}] * self.obj_ring.replicas
get_headers = slo_headers + [{}] * (len(codes) - len(slo_headers))
headers = {'headers': get_headers}
with set_http_connect(*codes, **headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
self.assertEqual(req.environ['QUERY_STRING'], '')
self.assertTrue('swift.post_as_copy' in req.environ)
def test_POST_delete_at(self):
t = str(int(time.time() + 100))
req = swob.Request.blank('/v1/a/c/o', method='POST',
headers={'Content-Type': 'foo/bar',
'X-Delete-At': t})
post_headers = []
def capture_headers(ip, port, device, part, method, path, headers,
**kwargs):
if method == 'POST':
post_headers.append(headers)
x_newest_responses = [200] * self.obj_ring.replicas + \
[404] * self.obj_ring.max_more_nodes
post_resp = [200] * self.obj_ring.replicas
codes = x_newest_responses + post_resp
with set_http_connect(*codes, give_connect=capture_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertEqual(req.environ['QUERY_STRING'], '') # sanity
self.assertTrue('swift.post_as_copy' in req.environ)
for given_headers in post_headers:
self.assertEqual(given_headers.get('X-Delete-At'), t)
self.assertTrue('X-Delete-At-Host' in given_headers)
self.assertTrue('X-Delete-At-Device' in given_headers)
self.assertTrue('X-Delete-At-Partition' in given_headers)
self.assertTrue('X-Delete-At-Container' in given_headers)
def test_PUT_delete_at(self):
t = str(int(time.time() + 100))
req = swob.Request.blank('/v1/a/c/o', method='PUT', body='',
@ -1000,43 +944,6 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
def test_COPY_simple(self):
req = swift.common.swob.Request.blank(
'/v1/a/c/o', method='COPY',
headers={'Content-Length': 0,
'Destination': 'c/o-copy'})
head_resp = [200] * self.obj_ring.replicas + \
[404] * self.obj_ring.max_more_nodes
put_resp = [201] * self.obj_ring.replicas
codes = head_resp + put_resp
with set_http_connect(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
def test_PUT_log_info(self):
req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT')
req.headers['x-copy-from'] = 'some/where'
req.headers['Content-Length'] = 0
# override FakeConn default resp headers to keep log_info clean
resp_headers = {'x-delete-at': None}
head_resp = [200] * self.obj_ring.replicas + \
[404] * self.obj_ring.max_more_nodes
put_resp = [201] * self.obj_ring.replicas
codes = head_resp + put_resp
with set_http_connect(*codes, headers=resp_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
self.assertEqual(
req.environ.get('swift.log_info'), ['x-copy-from:some/where'])
# and then check that we don't do that for originating POSTs
req = swift.common.swob.Request.blank('/v1/a/c/o')
req.method = 'POST'
req.headers['x-copy-from'] = 'else/where'
with set_http_connect(*codes, headers=resp_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
self.assertEqual(req.environ.get('swift.log_info'), None)
@patch_policies(
[StoragePolicy(0, '1-replica', True),
@ -1397,7 +1304,7 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
def test_PUT_ec_error_during_transfer_data(self):
class FakeReader(object):
def read(self, size):
raise exceptions.ChunkReadError('exception message')
raise IOError('error message')
req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
body='test body')
@ -1603,72 +1510,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
def test_COPY_cross_policy_type_from_replicated(self):
self.app.per_container_info = {
'c1': self.app.container_info.copy(),
'c2': self.app.container_info.copy(),
}
# make c2 use replicated storage policy 1
self.app.per_container_info['c2']['storage_policy'] = '1'
# a put request with copy from source c2
req = swift.common.swob.Request.blank('/v1/a/c1/o', method='PUT',
body='', headers={
'X-Copy-From': 'c2/o'})
# c2 get
codes = [200] * self.replicas(POLICIES[1])
codes += [404] * POLICIES[1].object_ring.max_more_nodes
# c1 put
codes += [201] * self.replicas()
expect_headers = {
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
with set_http_connect(*codes, expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
def test_COPY_cross_policy_type_to_replicated(self):
self.app.per_container_info = {
'c1': self.app.container_info.copy(),
'c2': self.app.container_info.copy(),
}
# make c1 use replicated storage policy 1
self.app.per_container_info['c1']['storage_policy'] = '1'
# a put request with copy from source c2
req = swift.common.swob.Request.blank('/v1/a/c1/o', method='PUT',
body='', headers={
'X-Copy-From': 'c2/o'})
# c2 get
codes = [404, 200] * self.policy.ec_ndata
headers = {
'X-Object-Sysmeta-Ec-Content-Length': 0,
}
# c1 put
codes += [201] * self.replicas(POLICIES[1])
with set_http_connect(*codes, headers=headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
def test_COPY_cross_policy_type_unknown(self):
self.app.per_container_info = {
'c1': self.app.container_info.copy(),
'c2': self.app.container_info.copy(),
}
# make c1 use some made up storage policy index
self.app.per_container_info['c1']['storage_policy'] = '13'
# a COPY request of c2 with destination in c1
req = swift.common.swob.Request.blank('/v1/a/c2/o', method='COPY',
body='', headers={
'Destination': 'c1/o'})
with set_http_connect():
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
def _make_ec_archive_bodies(self, test_body, policy=None):
policy = policy or self.policy
segment_size = policy.ec_segment_size
@ -2378,40 +2219,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
def test_COPY_with_ranges(self):
req = swift.common.swob.Request.blank(
'/v1/a/c/o', method='COPY',
headers={'Destination': 'c1/o',
'Range': 'bytes=5-10'})
# turn a real body into fragments
segment_size = self.policy.ec_segment_size
real_body = ('asdf' * segment_size)[:-10]
# split it up into chunks
chunks = [real_body[x:x + segment_size]
for x in range(0, len(real_body), segment_size)]
# we need only first chunk to rebuild 5-10 range
fragments = self.policy.pyeclib_driver.encode(chunks[0])
fragment_payloads = []
fragment_payloads.append(fragments)
node_fragments = zip(*fragment_payloads)
self.assertEqual(len(node_fragments), self.replicas()) # sanity
headers = {'X-Object-Sysmeta-Ec-Content-Length': str(len(real_body))}
responses = [(200, ''.join(node_fragments[i]), headers)
for i in range(POLICIES.default.ec_ndata)]
responses += [(201, '', {})] * self.obj_ring.replicas
status_codes, body_iter, headers = zip(*responses)
expect_headers = {
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
with set_http_connect(*status_codes, body_iter=body_iter,
headers=headers, expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
def test_GET_with_invalid_ranges(self):
# real body size is segment_size - 10 (just 1 segment)
segment_size = self.policy.ec_segment_size
@ -2424,18 +2231,6 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
self._test_invalid_ranges('GET', real_body,
segment_size, '%s-' % (segment_size + 10))
def test_COPY_with_invalid_ranges(self):
# real body size is segment_size - 10 (just 1 segment)
segment_size = self.policy.ec_segment_size
real_body = ('a' * segment_size)[:-10]
# range is out of real body but in segment size
self._test_invalid_ranges('COPY', real_body,
segment_size, '%s-' % (segment_size - 10))
# range is out of both real body and segment size
self._test_invalid_ranges('COPY', real_body,
segment_size, '%s-' % (segment_size + 10))
def _test_invalid_ranges(self, method, real_body, segment_size, req_range):
# make a request with range starts from more than real size.
body_etag = md5(real_body).hexdigest()

950
test/unit/proxy/test_server.py
View File

File diff suppressed because it is too large Load Diff

91
test/unit/proxy/test_sysmeta.py
View File

@ -283,94 +283,3 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertNotInHeaders(resp, self.original_sysmeta_headers_2)
def test_sysmeta_not_updated_by_POST(self):
self.app.object_post_as_copy = False
self._test_sysmeta_not_updated_by_POST()
def test_sysmeta_not_updated_by_POST_as_copy(self):
self.app.object_post_as_copy = True
self._test_sysmeta_not_updated_by_POST()
def test_sysmeta_updated_by_COPY(self):
# check sysmeta is updated by a COPY in same way as user meta
path = '/v1/a/c/o'
dest = '/c/o2'
env = {'REQUEST_METHOD': 'PUT'}
hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1)
hdrs.update(self.original_meta_headers_2)
req = Request.blank(path, environ=env, headers=hdrs, body='x')
resp = req.get_response(self.app)
self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'COPY'}
hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers)
hdrs.update(self.new_meta_headers)
hdrs.update(self.bad_headers)
hdrs.update({'Destination': dest})
req = Request.blank(path, environ=env, headers=hdrs)
resp = req.get_response(self.app)
self._assertStatus(resp, 201)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertInHeaders(resp, self.original_sysmeta_headers_2)
self._assertInHeaders(resp, self.changed_meta_headers)
self._assertInHeaders(resp, self.new_meta_headers)
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers)
req = Request.blank('/v1/a/c/o2', environ={})
resp = req.get_response(self.app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertInHeaders(resp, self.original_sysmeta_headers_2)
self._assertInHeaders(resp, self.changed_meta_headers)
self._assertInHeaders(resp, self.new_meta_headers)
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers)
def test_sysmeta_updated_by_COPY_from(self):
# check sysmeta is updated by a COPY in same way as user meta
path = '/v1/a/c/o'
env = {'REQUEST_METHOD': 'PUT'}
hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1)
hdrs.update(self.original_meta_headers_2)
req = Request.blank(path, environ=env, headers=hdrs, body='x')
resp = req.get_response(self.app)
self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'PUT'}
hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers)
hdrs.update(self.new_meta_headers)
hdrs.update(self.bad_headers)
hdrs.update({'X-Copy-From': '/c/o'})
req = Request.blank('/v1/a/c/o2', environ=env, headers=hdrs, body='')
resp = req.get_response(self.app)
self._assertStatus(resp, 201)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertInHeaders(resp, self.original_sysmeta_headers_2)
self._assertInHeaders(resp, self.changed_meta_headers)
self._assertInHeaders(resp, self.new_meta_headers)
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers)
req = Request.blank('/v1/a/c/o2', environ={})
resp = req.get_response(self.app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertInHeaders(resp, self.original_sysmeta_headers_2)
self._assertInHeaders(resp, self.changed_meta_headers)
self._assertInHeaders(resp, self.new_meta_headers)
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers)