swift/test/unit/common/middleware/test_versioned_writes.py

# Copyright (c) 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import functools
import json
import os
import time
import unittest
from swift.common import swob
from swift.common.middleware import versioned_writes, copy
from swift.common.swob import Request
from test.unit.common.middleware.helpers import FakeSwift


class FakeCache(object):

    def __init__(self, val):
        if 'status' not in val:
            val['status'] = 200
        self.val = val

    def get(self, *args):
        return self.val


def local_tz(func):
    '''
    Decorator to change the timezone when running a test.

    This uses the Eastern Time Zone definition from the time module's docs.
    Note that the timezone affects things like time.time() and time.mktime().
    '''
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        tz = os.environ.get('TZ', '')
        try:
            os.environ['TZ'] = 'EST+05EDT,M4.1.0,M10.5.0'
            time.tzset()
            return func(*args, **kwargs)
        finally:
            os.environ['TZ'] = tz
            time.tzset()
    return wrapper


class VersionedWritesBaseTestCase(unittest.TestCase):
    def setUp(self):
        self.app = FakeSwift()
        conf = {'allow_versioned_writes': 'true'}
        self.vw = versioned_writes.filter_factory(conf)(self.app)

    def call_app(self, req, app=None, expect_exception=False):
        if app is None:
            app = self.app

        self.authorized = []

        def authorize(req):
            self.authorized.append(req)

        if 'swift.authorize' not in req.environ:
            req.environ['swift.authorize'] = authorize

        req.headers.setdefault("User-Agent", "Marula Kruger")

        status = [None]
        headers = [None]

        def start_response(s, h, ei=None):
            status[0] = s
            headers[0] = h

        body_iter = app(req.environ, start_response)
        body = ''
        caught_exc = None
        try:
            for chunk in body_iter:
                body += chunk
        except Exception as exc:
            if expect_exception:
                caught_exc = exc
            else:
                raise

        if expect_exception:
            return status[0], headers[0], body, caught_exc
        else:
            return status[0], headers[0], body

    def call_vw(self, req, **kwargs):
        return self.call_app(req, app=self.vw, **kwargs)

    def assertRequestEqual(self, req, other):
        self.assertEqual(req.method, other.method)
        self.assertEqual(req.path, other.path)


class VersionedWritesTestCase(VersionedWritesBaseTestCase):
    def test_put_container(self):
        self.app.register('PUT', '/v1/a/c', swob.HTTPOk, {}, 'passed')
        req = Request.blank('/v1/a/c',
                            headers={'X-Versions-Location': 'ver_cont'},
                            environ={'REQUEST_METHOD': 'PUT'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')

        # check for sysmeta header
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[0]
        self.assertEqual('PUT', method)
        self.assertEqual('/v1/a/c', path)
        self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_container_allow_versioned_writes_false(self):
        self.vw.conf = {'allow_versioned_writes': 'false'}

        # PUT/POST container must fail as 412 when allow_versioned_writes
        # set to false
        for method in ('PUT', 'POST'):
            req = Request.blank('/v1/a/c',
                                headers={'X-Versions-Location': 'ver_cont'},
                                environ={'REQUEST_METHOD': method})
            status, headers, body = self.call_vw(req)
            self.assertEqual(status, "412 Precondition Failed")

        # GET performs as normal
        self.app.register('GET', '/v1/a/c', swob.HTTPOk, {}, 'passed')

        for method in ('GET', 'HEAD'):
            req = Request.blank('/v1/a/c',
                                headers={'X-Versions-Location': 'ver_cont'},
                                environ={'REQUEST_METHOD': method})
            status, headers, body = self.call_vw(req)
            self.assertEqual(status, '200 OK')

    def test_remove_versions_location(self):
        self.app.register('POST', '/v1/a/c', swob.HTTPOk, {}, 'passed')
        req = Request.blank('/v1/a/c',
                            headers={'X-Remove-Versions-Location': 'x'},
                            environ={'REQUEST_METHOD': 'POST'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')

        # check for sysmeta header
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[0]
        self.assertEqual('POST', method)
        self.assertEqual('/v1/a/c', path)
        self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
        self.assertEqual('',
                         req_headers['x-container-sysmeta-versions-location'])
        self.assertTrue('x-versions-location' in req_headers)
        self.assertEqual('', req_headers['x-versions-location'])
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_empty_versions_location(self):
        self.app.register('POST', '/v1/a/c', swob.HTTPOk, {}, 'passed')
        req = Request.blank('/v1/a/c',
                            headers={'X-Versions-Location': ''},
                            environ={'REQUEST_METHOD': 'POST'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')

        # check for sysmeta header
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[0]
        self.assertEqual('POST', method)
        self.assertEqual('/v1/a/c', path)
        self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
        self.assertEqual('',
                         req_headers['x-container-sysmeta-versions-location'])
        self.assertTrue('x-versions-location' in req_headers)
        self.assertEqual('', req_headers['x-versions-location'])
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_remove_add_versions_precedence(self):
        self.app.register(
            'POST', '/v1/a/c', swob.HTTPOk,
            {'x-container-sysmeta-versions-location': 'ver_cont'},
            'passed')
        req = Request.blank('/v1/a/c',
                            headers={'X-Remove-Versions-Location': 'x',
                                     'X-Versions-Location': 'ver_cont'},
                            environ={'REQUEST_METHOD': 'POST'})

        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertTrue(('X-Versions-Location', 'ver_cont') in headers)

        # check for sysmeta header
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[0]
        self.assertEqual('POST', method)
        self.assertEqual('/v1/a/c', path)
        self.assertTrue('x-container-sysmeta-versions-location' in req_headers)
        self.assertTrue('x-remove-versions-location' not in req_headers)
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_get_container(self):
        self.app.register(
            'GET', '/v1/a/c', swob.HTTPOk,
            {'x-container-sysmeta-versions-location': 'ver_cont'}, None)
        req = Request.blank(
            '/v1/a/c',
            environ={'REQUEST_METHOD': 'GET'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertTrue(('X-Versions-Location', 'ver_cont') in headers)
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_get_head(self):
        self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, None)
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'GET'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

        self.app.register('HEAD', '/v1/a/c/o', swob.HTTPOk, {}, None)
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'HEAD'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_put_object_no_versioning(self):
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')

        cache = FakeCache({})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

    def test_put_object_post_as_copy(self):
        # PUTs due to a post-as-copy should NOT cause a versioning op
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed')

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100',
                     'swift.post_as_copy': True})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '201 Created')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(1, self.app.call_count)

    def test_put_first_object_success(self):
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'GET', '/v1/a/c/o', swob.HTTPNotFound, {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100',
                     'swift.trans_id': 'fake_trans_id'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(2, self.app.call_count)
        self.assertEqual(['VW', None], self.app.swift_sources)
        self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids))

    def test_put_object_no_versioning_with_container_config_true(self):
        # set False to versions_write and expect no GET occurred
        self.vw.conf = {'allow_versioned_writes': 'false'}
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed')
        cache = FakeCache({'versions': 'ver_cont'})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '201 Created')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        called_method = [method for (method, path, hdrs) in self.app._calls]
        self.assertTrue('GET' not in called_method)

    def test_put_request_is_dlo_manifest_with_container_config_true(self):
        # set x-object-manifest on request and expect no versioning occurred
        # only the PUT for the original client request
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed')
        cache = FakeCache({'versions': 'ver_cont'})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        req.headers['X-Object-Manifest'] = 'req/manifest'
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '201 Created')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(1, self.app.call_count)

    def test_put_version_is_dlo_manifest_with_container_config_true(self):
        # set x-object-manifest on response and expect no versioning occurred
        # only initial GET on source object ok followed by PUT
        self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
                          {'X-Object-Manifest': 'resp/manifest'}, 'passed')
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed')
        cache = FakeCache({'versions': 'ver_cont'})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '201 Created')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(2, self.app.call_count)

    def test_delete_object_no_versioning_with_container_config_true(self):
        # set False to versions_write obviously and expect no GET versioning
        # container and GET/PUT called (just delete object as normal)
        self.vw.conf = {'allow_versioned_writes': 'false'}
        self.app.register(
            'DELETE', '/v1/a/c/o', swob.HTTPNoContent, {}, 'passed')
        cache = FakeCache({'versions': 'ver_cont'})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '204 No Content')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        called_method = \
            [method for (method, path, rheaders) in self.app._calls]
        self.assertTrue('PUT' not in called_method)
        self.assertTrue('GET' not in called_method)
        self.assertEqual(1, self.app.call_count)

    def test_new_version_success(self):
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, 'passed')
        self.app.register(
            'GET', '/v1/a/c/o', swob.HTTPOk,
            {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed')
        self.app.register(
            'PUT', '/v1/a/ver_cont/001o/0000000001.00000', swob.HTTPCreated,
            {}, None)
        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100',
                     'swift.trans_id': 'fake_trans_id'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '201 Created')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(['VW', 'VW', None], self.app.swift_sources)
        self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids))

    def test_new_version_get_errors(self):
        # GET on source fails, expect client error response,
        # no PUT should happen
        self.app.register(
            'GET', '/v1/a/c/o', swob.HTTPBadRequest, {}, None)
        cache = FakeCache({'versions': 'ver_cont'})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '412 Precondition Failed')
        self.assertEqual(1, self.app.call_count)

        # GET on source fails, expect server error response
        self.app.register(
            'GET', '/v1/a/c/o', swob.HTTPBadGateway, {}, None)
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '503 Service Unavailable')
        self.assertEqual(2, self.app.call_count)

    def test_new_version_put_errors(self):
        # PUT of version fails, expect client error response
        self.app.register(
            'GET', '/v1/a/c/o', swob.HTTPOk,
            {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed')
        self.app.register(
            'PUT', '/v1/a/ver_cont/001o/0000000001.00000',
            swob.HTTPUnauthorized, {}, None)
        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '412 Precondition Failed')
        self.assertEqual(2, self.app.call_count)

        # PUT of version fails, expect server error response
        self.app.register(
            'PUT', '/v1/a/ver_cont/001o/0000000001.00000', swob.HTTPBadGateway,
            {}, None)
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '503 Service Unavailable')
        self.assertEqual(4, self.app.call_count)

    @local_tz
    def test_new_version_sysmeta_precedence(self):
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'GET', '/v1/a/c/o', swob.HTTPOk,
            {'last-modified': 'Thu, 1 Jan 1970 00:00:00 GMT'}, 'passed')
        self.app.register(
            'PUT', '/v1/a/ver_cont/001o/0000000000.00000', swob.HTTPOk,
            {}, None)

        # fill cache with two different values for versions location
        # new middleware should use sysmeta first
        cache = FakeCache({'versions': 'old_ver_cont',
                          'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

        # check that sysmeta header was used
        calls = self.app.calls_with_headers
        method, path, req_headers = calls[1]
        self.assertEqual('PUT', method)
        self.assertEqual('/v1/a/ver_cont/001o/0000000000.00000', path)

    def test_delete_first_object_success(self):
        self.app.register(
            'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'GET',
            '/v1/a/ver_cont?format=json&prefix=001o/&marker=&reverse=on',
            swob.HTTPNotFound, {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0', 'swift.trans_id': 'fake_trans_id'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(2, self.app.call_count)
        self.assertEqual(['VW', None], self.app.swift_sources)
        self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids))

        prefix_listing_prefix = '/v1/a/ver_cont?format=json&prefix=001o/&'
        self.assertEqual(self.app.calls, [
            ('GET', prefix_listing_prefix + 'marker=&reverse=on'),
            ('DELETE', '/v1/a/c/o'),
        ])

    def test_delete_latest_version_success(self):
        self.app.register(
            'GET',
            '/v1/a/ver_cont?format=json&prefix=001o/&marker=&reverse=on',
            swob.HTTPOk, {},
            '[{"hash": "y", '
            '"last_modified": "2014-11-21T14:23:02.206740", '
            '"bytes": 3, '
            '"name": "001o/2", '
            '"content_type": "text/plain"}, '
            '{"hash": "x", '
            '"last_modified": "2014-11-21T14:14:27.409100", '
            '"bytes": 3, '
            '"name": "001o/1", '
            '"content_type": "text/plain"}]')
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/2', swob.HTTPCreated,
            {'content-length': '3'}, None)
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, None)
        self.app.register(
            'DELETE', '/v1/a/ver_cont/001o/2', swob.HTTPOk,
            {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            headers={'X-If-Delete-At': 1},
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0', 'swift.trans_id': 'fake_trans_id'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(4, self.app.call_count)
        self.assertEqual(['VW', 'VW', 'VW', 'VW'], self.app.swift_sources)
        self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids))

        # check that X-If-Delete-At was removed from DELETE request
        req_headers = self.app.headers[-1]
        self.assertNotIn('x-if-delete-at', [h.lower() for h in req_headers])

        prefix_listing_prefix = '/v1/a/ver_cont?format=json&prefix=001o/&'
        self.assertEqual(self.app.calls, [
            ('GET', prefix_listing_prefix + 'marker=&reverse=on'),
            ('GET', '/v1/a/ver_cont/001o/2'),
            ('PUT', '/v1/a/c/o'),
            ('DELETE', '/v1/a/ver_cont/001o/2'),
        ])

    def test_delete_single_version_success(self):
        # check that if the first listing page has just a single item then
        # it is not erroneously inferred to be a non-reversed listing
        self.app.register(
            'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'GET',
            '/v1/a/ver_cont?format=json&prefix=001o/&marker=&reverse=on',
            swob.HTTPOk, {},
            '[{"hash": "y", '
            '"last_modified": "2014-11-21T14:23:02.206740", '
            '"bytes": 3, '
            '"name": "001o/1", '
            '"content_type": "text/plain"}]')
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/1', swob.HTTPOk,
            {'content-length': '3'}, None)
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, None)
        self.app.register(
            'DELETE', '/v1/a/ver_cont/001o/1', swob.HTTPOk,
            {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])

        prefix_listing_prefix = '/v1/a/ver_cont?format=json&prefix=001o/&'
        self.assertEqual(self.app.calls, [
            ('GET', prefix_listing_prefix + 'marker=&reverse=on'),
            ('GET', '/v1/a/ver_cont/001o/1'),
            ('PUT', '/v1/a/c/o'),
            ('DELETE', '/v1/a/ver_cont/001o/1'),
        ])

    def test_DELETE_on_expired_versioned_object(self):
        self.app.register(
            'GET',
            '/v1/a/ver_cont?format=json&prefix=001o/&marker=&reverse=on',
            swob.HTTPOk, {},
            '[{"hash": "y", '
            '"last_modified": "2014-11-21T14:23:02.206740", '
            '"bytes": 3, '
            '"name": "001o/2", '
            '"content_type": "text/plain"}, '
            '{"hash": "x", '
            '"last_modified": "2014-11-21T14:14:27.409100", '
            '"bytes": 3, '
            '"name": "001o/1", '
            '"content_type": "text/plain"}]')

        # expired object
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/2', swob.HTTPNotFound,
            {}, None)
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/1', swob.HTTPCreated,
            {'content-length': '3'}, None)
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPOk, {}, None)
        self.app.register(
            'DELETE', '/v1/a/ver_cont/001o/1', swob.HTTPOk,
            {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(5, self.app.call_count)

        prefix_listing_prefix = '/v1/a/ver_cont?format=json&prefix=001o/&'
        self.assertEqual(self.app.calls, [
            ('GET', prefix_listing_prefix + 'marker=&reverse=on'),
            ('GET', '/v1/a/ver_cont/001o/2'),
            ('GET', '/v1/a/ver_cont/001o/1'),
            ('PUT', '/v1/a/c/o'),
            ('DELETE', '/v1/a/ver_cont/001o/1'),
        ])

    def test_denied_DELETE_of_versioned_object(self):
        authorize_call = []
        self.app.register(
            'GET',
            '/v1/a/ver_cont?format=json&prefix=001o/&marker=&reverse=on',
            swob.HTTPOk, {},
            '[{"hash": "y", '
            '"last_modified": "2014-11-21T14:23:02.206740", '
            '"bytes": 3, '
            '"name": "001o/2", '
            '"content_type": "text/plain"}, '
            '{"hash": "x", '
            '"last_modified": "2014-11-21T14:14:27.409100", '
            '"bytes": 3, '
            '"name": "001o/1", '
            '"content_type": "text/plain"}]')

        def fake_authorize(req):
            # the container GET is pre-auth'd so here we deny the object DELETE
            authorize_call.append(req)
            return swob.HTTPForbidden()

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'swift.authorize': fake_authorize,
                     'CONTENT_LENGTH': '0'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '403 Forbidden')
        self.assertEqual(len(authorize_call), 1)
        self.assertRequestEqual(req, authorize_call[0])

        prefix_listing_prefix = '/v1/a/ver_cont?format=json&prefix=001o/&'
        self.assertEqual(self.app.calls, [
            ('GET', prefix_listing_prefix + 'marker=&reverse=on'),
        ])


class VersionedWritesOldContainersTestCase(VersionedWritesBaseTestCase):
    def test_delete_latest_version_success(self):
        self.app.register(
            'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=&reverse=on',
            swob.HTTPOk, {},
            '[{"hash": "x", '
            '"last_modified": "2014-11-21T14:14:27.409100", '
            '"bytes": 3, '
            '"name": "001o/1", '
            '"content_type": "text/plain"}, '
            '{"hash": "y", '
            '"last_modified": "2014-11-21T14:23:02.206740", '
            '"bytes": 3, '
            '"name": "001o/2", '
            '"content_type": "text/plain"}]')
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/'
            '&marker=001o/2',
            swob.HTTPNotFound, {}, None)
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/2', swob.HTTPCreated,
            {'content-length': '3'}, None)
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, None)
        self.app.register(
            'DELETE', '/v1/a/ver_cont/001o/2', swob.HTTPOk,
            {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            headers={'X-If-Delete-At': 1},
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0', 'swift.trans_id': 'fake_trans_id'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(5, self.app.call_count)
        self.assertEqual(['VW', 'VW', 'VW', 'VW', 'VW'],
                         self.app.swift_sources)
        self.assertEqual({'fake_trans_id'}, set(self.app.txn_ids))

        # check that X-If-Delete-At was removed from DELETE request
        req_headers = self.app.headers[-1]
        self.assertNotIn('x-if-delete-at', [h.lower() for h in req_headers])

        prefix_listing_prefix = '/v1/a/ver_cont?format=json&prefix=001o/&'
        self.assertEqual(self.app.calls, [
            ('GET', prefix_listing_prefix + 'marker=&reverse=on'),
            ('GET', prefix_listing_prefix + 'marker=001o/2'),
            ('GET', '/v1/a/ver_cont/001o/2'),
            ('PUT', '/v1/a/c/o'),
            ('DELETE', '/v1/a/ver_cont/001o/2'),
        ])

    def test_DELETE_on_expired_versioned_object(self):
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=&reverse=on',
            swob.HTTPOk, {},
            '[{"hash": "x", '
            '"last_modified": "2014-11-21T14:14:27.409100", '
            '"bytes": 3, '
            '"name": "001o/1", '
            '"content_type": "text/plain"}, '
            '{"hash": "y", '
            '"last_modified": "2014-11-21T14:23:02.206740", '
            '"bytes": 3, '
            '"name": "001o/2", '
            '"content_type": "text/plain"}]')
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/'
            '&marker=001o/2',
            swob.HTTPNotFound, {}, None)

        # expired object
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/2', swob.HTTPNotFound,
            {}, None)
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/1', swob.HTTPCreated,
            {'content-length': '3'}, None)
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPOk, {}, None)
        self.app.register(
            'DELETE', '/v1/a/ver_cont/001o/1', swob.HTTPOk,
            {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '200 OK')
        self.assertEqual(len(self.authorized), 1)
        self.assertRequestEqual(req, self.authorized[0])
        self.assertEqual(6, self.app.call_count)

        prefix_listing_prefix = '/v1/a/ver_cont?format=json&prefix=001o/&'
        self.assertEqual(self.app.calls, [
            ('GET', prefix_listing_prefix + 'marker=&reverse=on'),
            ('GET', prefix_listing_prefix + 'marker=001o/2'),
            ('GET', '/v1/a/ver_cont/001o/2'),
            ('GET', '/v1/a/ver_cont/001o/1'),
            ('PUT', '/v1/a/c/o'),
            ('DELETE', '/v1/a/ver_cont/001o/1'),
        ])

    def test_denied_DELETE_of_versioned_object(self):
        authorize_call = []
        self.app.register(
            'DELETE', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=&reverse=on',
            swob.HTTPOk, {},
            '[{"hash": "x", '
            '"last_modified": "2014-11-21T14:14:27.409100", '
            '"bytes": 3, '
            '"name": "001o/1", '
            '"content_type": "text/plain"}, '
            '{"hash": "y", '
            '"last_modified": "2014-11-21T14:23:02.206740", '
            '"bytes": 3, '
            '"name": "001o/2", '
            '"content_type": "text/plain"}]')
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/'
            '&marker=001o/2',
            swob.HTTPNotFound, {}, None)
        self.app.register(
            'DELETE', '/v1/a/c/o', swob.HTTPForbidden,
            {}, None)

        def fake_authorize(req):
            authorize_call.append(req)
            return swob.HTTPForbidden()

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'swift.authorize': fake_authorize,
                     'CONTENT_LENGTH': '0'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '403 Forbidden')
        self.assertEqual(len(authorize_call), 1)
        self.assertRequestEqual(req, authorize_call[0])
        prefix_listing_prefix = '/v1/a/ver_cont?format=json&prefix=001o/&'
        self.assertEqual(self.app.calls, [
            ('GET', prefix_listing_prefix + 'marker=&reverse=on'),
            ('GET', prefix_listing_prefix + 'marker=001o/2'),
        ])

    def test_partially_upgraded_cluster(self):
        old_versions = [
            {'hash': 'etag%d' % x,
             'last_modified': "2014-11-21T14:14:%02d.409100" % x,
             'bytes': 3,
             'name': '001o/%d' % x,
             'content_type': 'text/plain'}
            for x in range(5)]

        # first container server can reverse
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=&reverse=on',
            swob.HTTPOk, {}, json.dumps(list(reversed(old_versions[2:]))))
        # but all objects are already gone
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/4', swob.HTTPNotFound,
            {}, None)
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/3', swob.HTTPNotFound,
            {}, None)
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/2', swob.HTTPNotFound,
            {}, None)

        # second container server can't reverse
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=001o/2&reverse=on',
            swob.HTTPOk, {}, json.dumps(old_versions[3:]))

        # subsequent requests shouldn't reverse
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=&end_marker=001o/2',
            swob.HTTPOk, {}, json.dumps(old_versions[:1]))
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=001o/0&end_marker=001o/2',
            swob.HTTPOk, {}, json.dumps(old_versions[1:2]))
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=001o/1&end_marker=001o/2',
            swob.HTTPOk, {}, '[]')
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/1', swob.HTTPOk,
            {'content-length': '3'}, None)
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, None)
        self.app.register(
            'DELETE', '/v1/a/ver_cont/001o/1', swob.HTTPNoContent,
            {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '204 No Content')
        prefix_listing_prefix = '/v1/a/ver_cont?format=json&prefix=001o/&'
        self.assertEqual(self.app.calls, [
            ('GET', prefix_listing_prefix + 'marker=&reverse=on'),
            ('GET', '/v1/a/ver_cont/001o/4'),
            ('GET', '/v1/a/ver_cont/001o/3'),
            ('GET', '/v1/a/ver_cont/001o/2'),
            ('GET', prefix_listing_prefix + 'marker=001o/2&reverse=on'),
            ('GET', prefix_listing_prefix + 'marker=&end_marker=001o/2'),
            ('GET', prefix_listing_prefix + 'marker=001o/0&end_marker=001o/2'),
            ('GET', prefix_listing_prefix + 'marker=001o/1&end_marker=001o/2'),
            ('GET', '/v1/a/ver_cont/001o/1'),
            ('PUT', '/v1/a/c/o'),
            ('DELETE', '/v1/a/ver_cont/001o/1'),
        ])

    def test_partially_upgraded_cluster_single_result_on_second_page(self):
        old_versions = [
            {'hash': 'etag%d' % x,
             'last_modified': "2014-11-21T14:14:%02d.409100" % x,
             'bytes': 3,
             'name': '001o/%d' % x,
             'content_type': 'text/plain'}
            for x in range(5)]

        # first container server can reverse
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=&reverse=on',
            swob.HTTPOk, {}, json.dumps(list(reversed(old_versions[-2:]))))
        # but both objects are already gone
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/4', swob.HTTPNotFound,
            {}, None)
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/3', swob.HTTPNotFound,
            {}, None)

        # second container server can't reverse
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=001o/3&reverse=on',
            swob.HTTPOk, {}, json.dumps(old_versions[4:]))

        # subsequent requests shouldn't reverse
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=&end_marker=001o/3',
            swob.HTTPOk, {}, json.dumps(old_versions[:2]))
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=001o/1&end_marker=001o/3',
            swob.HTTPOk, {}, json.dumps(old_versions[2:3]))
        self.app.register(
            'GET', '/v1/a/ver_cont?format=json&prefix=001o/&'
            'marker=001o/2&end_marker=001o/3',
            swob.HTTPOk, {}, '[]')
        self.app.register(
            'GET', '/v1/a/ver_cont/001o/2', swob.HTTPOk,
            {'content-length': '3'}, None)
        self.app.register(
            'PUT', '/v1/a/c/o', swob.HTTPCreated, {}, None)
        self.app.register(
            'DELETE', '/v1/a/ver_cont/001o/2', swob.HTTPNoContent,
            {}, None)

        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/c/o',
            environ={'REQUEST_METHOD': 'DELETE', 'swift.cache': cache,
                     'CONTENT_LENGTH': '0'})
        status, headers, body = self.call_vw(req)
        self.assertEqual(status, '204 No Content')
        prefix_listing_prefix = '/v1/a/ver_cont?format=json&prefix=001o/&'
        self.assertEqual(self.app.calls, [
            ('GET', prefix_listing_prefix + 'marker=&reverse=on'),
            ('GET', '/v1/a/ver_cont/001o/4'),
            ('GET', '/v1/a/ver_cont/001o/3'),
            ('GET', prefix_listing_prefix + 'marker=001o/3&reverse=on'),
            ('GET', prefix_listing_prefix + 'marker=&end_marker=001o/3'),
            ('GET', prefix_listing_prefix + 'marker=001o/1&end_marker=001o/3'),
            ('GET', prefix_listing_prefix + 'marker=001o/2&end_marker=001o/3'),
            ('GET', '/v1/a/ver_cont/001o/2'),
            ('PUT', '/v1/a/c/o'),
            ('DELETE', '/v1/a/ver_cont/001o/2'),
        ])


class VersionedWritesCopyingTestCase(VersionedWritesBaseTestCase):
    # verify interaction of copy and versioned_writes middlewares

    def setUp(self):
        self.app = FakeSwift()
        conf = {'allow_versioned_writes': 'true'}
        self.vw = versioned_writes.filter_factory(conf)(self.app)
        self.filter = copy.filter_factory({})(self.vw)

    def call_filter(self, req, **kwargs):
        return self.call_app(req, app=self.filter, **kwargs)

    def test_copy_first_version(self):
        # no existing object to move to the versions container
        self.app.register(
            'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPNotFound, {}, None)
        self.app.register(
            'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed')
        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/src_cont/src_obj',
            environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'},
            headers={'Destination': 'tgt_cont/tgt_obj'})
        status, headers, body = self.call_filter(req)
        self.assertEqual(status, '201 Created')
        self.assertEqual(len(self.authorized), 2)
        self.assertEqual('GET', self.authorized[0].method)
        self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
        self.assertEqual('PUT', self.authorized[1].method)
        self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
        # note the GET on tgt_cont/tgt_obj is pre-authed
        self.assertEqual(3, self.app.call_count, self.app.calls)

    def test_copy_new_version(self):
        # existing object should be moved to versions container
        self.app.register(
            'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'GET', '/v1/a/tgt_cont/tgt_obj', swob.HTTPOk,
            {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed')
        self.app.register(
            'PUT', '/v1/a/ver_cont/007tgt_obj/0000000001.00000', swob.HTTPOk,
            {}, None)
        self.app.register(
            'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed')
        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/a/src_cont/src_obj',
            environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'},
            headers={'Destination': 'tgt_cont/tgt_obj'})
        status, headers, body = self.call_filter(req)
        self.assertEqual(status, '201 Created')
        self.assertEqual(len(self.authorized), 2)
        self.assertEqual('GET', self.authorized[0].method)
        self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
        self.assertEqual('PUT', self.authorized[1].method)
        self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
        self.assertEqual(4, self.app.call_count)

    def test_copy_new_version_different_account(self):
        self.app.register(
            'GET', '/v1/src_a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'GET', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPOk,
            {'last-modified': 'Thu, 1 Jan 1970 00:00:01 GMT'}, 'passed')
        self.app.register(
            'PUT', '/v1/tgt_a/ver_cont/007tgt_obj/0000000001.00000',
            swob.HTTPOk, {}, None)
        self.app.register(
            'PUT', '/v1/tgt_a/tgt_cont/tgt_obj', swob.HTTPCreated, {},
            'passed')
        cache = FakeCache({'sysmeta': {'versions-location': 'ver_cont'}})
        req = Request.blank(
            '/v1/src_a/src_cont/src_obj',
            environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache,
                     'CONTENT_LENGTH': '100'},
            headers={'Destination': 'tgt_cont/tgt_obj',
                     'Destination-Account': 'tgt_a'})
        status, headers, body = self.call_filter(req)
        self.assertEqual(status, '201 Created')
        self.assertEqual(len(self.authorized), 2)
        self.assertEqual('GET', self.authorized[0].method)
        self.assertEqual('/v1/src_a/src_cont/src_obj', self.authorized[0].path)
        self.assertEqual('PUT', self.authorized[1].method)
        self.assertEqual('/v1/tgt_a/tgt_cont/tgt_obj', self.authorized[1].path)
        self.assertEqual(4, self.app.call_count)

    def test_copy_object_no_versioning_with_container_config_true(self):
        # set False to versions_write obviously and expect no extra
        # COPY called (just copy object as normal)
        self.vw.conf = {'allow_versioned_writes': 'false'}
        self.app.register(
            'GET', '/v1/a/src_cont/src_obj', swob.HTTPOk, {}, 'passed')
        self.app.register(
            'PUT', '/v1/a/tgt_cont/tgt_obj', swob.HTTPCreated, {}, 'passed')
        cache = FakeCache({'versions': 'ver_cont'})
        req = Request.blank(
            '/v1/a/src_cont/src_obj',
            environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache},
            headers={'Destination': '/tgt_cont/tgt_obj'})
        status, headers, body = self.call_filter(req)
        self.assertEqual(status, '201 Created')
        self.assertEqual(len(self.authorized), 2)
        self.assertEqual('GET', self.authorized[0].method)
        self.assertEqual('/v1/a/src_cont/src_obj', self.authorized[0].path)
        self.assertEqual('PUT', self.authorized[1].method)
        self.assertEqual('/v1/a/tgt_cont/tgt_obj', self.authorized[1].path)
        self.assertEqual(2, self.app.call_count)