swift/test/unit/account/test_utils.py

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools
import time
import unittest
import json

import mock

from swift.account import utils, backend
from swift.common.storage_policy import POLICIES, StoragePolicy
from swift.common.swob import Request
from swift.common.utils import Timestamp
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.request_helpers import get_reserved_name

from test.unit import patch_policies, make_timestamp_iter
from test.unit.common.test_db import TestDbBase


class TestFakeAccountBroker(unittest.TestCase):

    def test_fake_broker_get_info(self):
        broker = utils.FakeAccountBroker()
        now = time.time()
        with mock.patch('time.time', new=lambda: now):
            info = broker.get_info()
        timestamp = Timestamp(now)
        expected = {
            'container_count': 0,
            'object_count': 0,
            'bytes_used': 0,
            'created_at': timestamp.internal,
            'put_timestamp': timestamp.internal,
        }
        self.assertEqual(info, expected)

    def test_fake_broker_list_containers_iter(self):
        broker = utils.FakeAccountBroker()
        self.assertEqual(broker.list_containers_iter(), [])

    def test_fake_broker_metadata(self):
        broker = utils.FakeAccountBroker()
        self.assertEqual(broker.metadata, {})

    def test_fake_broker_get_policy_stats(self):
        broker = utils.FakeAccountBroker()
        self.assertEqual(broker.get_policy_stats(), {})


class TestAccountUtils(TestDbBase):
    server_type = 'account'

    def setUp(self):
        super(TestAccountUtils, self).setUp()
        self.ts = make_timestamp_iter()

    def test_get_response_headers_fake_broker(self):
        broker = utils.FakeAccountBroker()
        now = time.time()
        expected = {
            'X-Account-Container-Count': 0,
            'X-Account-Object-Count': 0,
            'X-Account-Bytes-Used': 0,
            'X-Timestamp': Timestamp(now).normal,
            'X-PUT-Timestamp': Timestamp(now).normal,
        }
        with mock.patch('time.time', new=lambda: now):
            resp_headers = utils.get_response_headers(broker)
        self.assertEqual(resp_headers, expected)

    def test_get_response_headers_empty_memory_broker(self):
        broker = backend.AccountBroker(self.db_path, account='a')
        now = time.time()
        with mock.patch('time.time', new=lambda: now):
            broker.initialize(Timestamp(now).internal)
        expected = {
            'X-Account-Container-Count': 0,
            'X-Account-Object-Count': 0,
            'X-Account-Bytes-Used': 0,
            'X-Timestamp': Timestamp(now).normal,
            'X-PUT-Timestamp': Timestamp(now).normal,
        }
        resp_headers = utils.get_response_headers(broker)
        self.assertEqual(resp_headers, expected)

    @patch_policies
    def test_get_response_headers_with_data(self):
        broker = backend.AccountBroker(self.db_path, account='a')
        now = time.time()
        with mock.patch('time.time', new=lambda: now):
            broker.initialize(Timestamp(now).internal)
        # add some container data
        ts = (Timestamp(t).internal for t in itertools.count(int(now)))
        total_containers = 0
        total_objects = 0
        total_bytes = 0
        for policy in POLICIES:
            delete_timestamp = next(ts)
            put_timestamp = next(ts)
            object_count = int(policy)
            bytes_used = int(policy) * 10
            broker.put_container('c-%s' % policy.name, put_timestamp,
                                 delete_timestamp, object_count, bytes_used,
                                 int(policy))
            total_containers += 1
            total_objects += object_count
            total_bytes += bytes_used
        expected = HeaderKeyDict({
            'X-Account-Container-Count': total_containers,
            'X-Account-Object-Count': total_objects,
            'X-Account-Bytes-Used': total_bytes,
            'X-Timestamp': Timestamp(now).normal,
            'X-PUT-Timestamp': Timestamp(now).normal,
        })
        for policy in POLICIES:
            prefix = 'X-Account-Storage-Policy-%s-' % policy.name
            expected[prefix + 'Container-Count'] = 1
            expected[prefix + 'Object-Count'] = int(policy)
            expected[prefix + 'Bytes-Used'] = int(policy) * 10
        resp_headers = utils.get_response_headers(broker)
        per_policy_container_headers = [
            h for h in resp_headers if
            h.lower().startswith('x-account-storage-policy-') and
            h.lower().endswith('-container-count')]
        self.assertTrue(per_policy_container_headers)
        for key, value in resp_headers.items():
            expected_value = expected.pop(key)
            self.assertEqual(expected_value, str(value),
                             'value for %r was %r not %r' % (
                                 key, value, expected_value))
        self.assertFalse(expected)

    @patch_policies
    def test_get_response_headers_with_legacy_data(self):
        broker = backend.AccountBroker(self.db_path, account='a')
        now = time.time()
        with mock.patch('time.time', new=lambda: now):
            broker.initialize(Timestamp(now).internal)
        # add some container data
        ts = (Timestamp(t).internal for t in itertools.count(int(now)))
        total_containers = 0
        total_objects = 0
        total_bytes = 0
        for policy in POLICIES:
            delete_timestamp = next(ts)
            put_timestamp = next(ts)
            object_count = int(policy)
            bytes_used = int(policy) * 10
            broker.put_container('c-%s' % policy.name, put_timestamp,
                                 delete_timestamp, object_count, bytes_used,
                                 int(policy))
            total_containers += 1
            total_objects += object_count
            total_bytes += bytes_used
        expected = HeaderKeyDict({
            'X-Account-Container-Count': total_containers,
            'X-Account-Object-Count': total_objects,
            'X-Account-Bytes-Used': total_bytes,
            'X-Timestamp': Timestamp(now).normal,
            'X-PUT-Timestamp': Timestamp(now).normal,
        })
        for policy in POLICIES:
            prefix = 'X-Account-Storage-Policy-%s-' % policy.name
            expected[prefix + 'Object-Count'] = int(policy)
            expected[prefix + 'Bytes-Used'] = int(policy) * 10
        orig_policy_stats = broker.get_policy_stats

        def stub_policy_stats(*args, **kwargs):
            policy_stats = orig_policy_stats(*args, **kwargs)
            for stats in policy_stats.values():
                # legacy db's won't return container_count
                del stats['container_count']
            return policy_stats
        broker.get_policy_stats = stub_policy_stats
        resp_headers = utils.get_response_headers(broker)
        per_policy_container_headers = [
            h for h in resp_headers if
            h.lower().startswith('x-account-storage-policy-') and
            h.lower().endswith('-container-count')]
        self.assertFalse(per_policy_container_headers)
        for key, value in resp_headers.items():
            expected_value = expected.pop(key)
            self.assertEqual(expected_value, str(value),
                             'value for %r was %r not %r' % (
                                 key, value, expected_value))
        self.assertFalse(expected)

    def test_account_listing_response(self):
        req = Request.blank('')
        now = time.time()
        with mock.patch('time.time', new=lambda: now):
            resp = utils.account_listing_response('a', req, 'text/plain')
        self.assertEqual(resp.status_int, 204)
        expected = HeaderKeyDict({
            'Content-Type': 'text/plain; charset=utf-8',
            'X-Account-Container-Count': 0,
            'X-Account-Object-Count': 0,
            'X-Account-Bytes-Used': 0,
            'X-Timestamp': Timestamp(now).normal,
            'X-PUT-Timestamp': Timestamp(now).normal,
        })
        self.assertEqual(expected, resp.headers)
        self.assertEqual(b'', resp.body)

    @patch_policies([StoragePolicy(0, 'zero', is_default=True)])
    def test_account_listing_reserved_names(self):
        broker = backend.AccountBroker(self.db_path, account='a')
        put_timestamp = next(self.ts)
        now = time.time()
        with mock.patch('time.time', new=lambda: now):
            broker.initialize(put_timestamp.internal)
        container_timestamp = next(self.ts)
        broker.put_container(get_reserved_name('foo'),
                             container_timestamp.internal, 0, 10, 100, 0)

        req = Request.blank('')
        resp = utils.account_listing_response(
            'a', req, 'application/json', broker)
        self.assertEqual(resp.status_int, 200)
        expected = HeaderKeyDict({
            'Content-Type': 'application/json; charset=utf-8',
            'Content-Length': 2,
            'X-Account-Container-Count': 1,
            'X-Account-Object-Count': 10,
            'X-Account-Bytes-Used': 100,
            'X-Timestamp': Timestamp(now).normal,
            'X-PUT-Timestamp': put_timestamp.normal,
            'X-Account-Storage-Policy-Zero-Container-Count': 1,
            'X-Account-Storage-Policy-Zero-Object-Count': 10,
            'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
        })
        self.assertEqual(expected, resp.headers)
        self.assertEqual(b'[]', resp.body)

        req = Request.blank('', headers={
            'X-Backend-Allow-Reserved-Names': 'true'})
        resp = utils.account_listing_response(
            'a', req, 'application/json', broker)
        self.assertEqual(resp.status_int, 200)
        expected = HeaderKeyDict({
            'Content-Type': 'application/json; charset=utf-8',
            'Content-Length': 97,
            'X-Account-Container-Count': 1,
            'X-Account-Object-Count': 10,
            'X-Account-Bytes-Used': 100,
            'X-Timestamp': Timestamp(now).normal,
            'X-PUT-Timestamp': put_timestamp.normal,
            'X-Account-Storage-Policy-Zero-Container-Count': 1,
            'X-Account-Storage-Policy-Zero-Object-Count': 10,
            'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
        })
        self.assertEqual(expected, resp.headers)
        expected = [{
            "last_modified": container_timestamp.isoformat,
            "count": 10,
            "bytes": 100,
            "name": get_reserved_name('foo'),
        }]
        self.assertEqual(sorted(json.dumps(expected).encode('ascii')),
                         sorted(resp.body))