ceb677d489
The swiftclient supports setting full_listing to True which will ignore the limit/marker parameters and internally do a while loop to retrieve all the containers or objects. We pass in full_listing=True to both get_container() and get_account() and in both cases that is bad, one reason it's bad is because it ignores any limit sent. Now that in itself is not bad since we dont use those parameters at all, in fact we rely on client side pagination in Angular using st-pagination for the hz-dynamic-table that lists all containers and objects. The bad part here is that with full_listing if we have a customer with 100k containers or 100k objects the Horizon REST API will try to gather all those resources and return it in the API response to the Angular client side code. This makes it easy for a end-user to starve Horizon of resources, create a container, upload 1M objects, go to Horizon and try to list the container and Horizon will after some refreshes hang because it's processing the requests for a long time or because it runs out of memory and crashes. This adds the configuration option SWIFT_PANEL_FULL_LISTING that defaults to True keeping the current behaviour but can be set to False by operators to prevent this issue until the Swift panel has been migrated to use correct pagination. Change-Id: Id41200aaeec3df4aff1ace887a42352728fc4419 Signed-off-by: Tobias Urdin <tobias.urdin@binero.com>
321 lines
13 KiB
Python
321 lines
13 KiB
Python
# Copyright 2012 United States Government as represented by the
|
|
# Administrator of the National Aeronautics and Space Administration.
|
|
# All Rights Reserved.
|
|
#
|
|
# Copyright 2012 Nebula, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
from unittest import mock
|
|
|
|
from django.test.utils import override_settings
|
|
|
|
from horizon import exceptions
|
|
|
|
from openstack_dashboard import api
|
|
from openstack_dashboard.test import helpers as test
|
|
|
|
|
|
@mock.patch('swiftclient.client.Connection')
|
|
class SwiftApiTests(test.APIMockTestCase):
|
|
def _test_swift_get_containers(self, mock_swiftclient, full_listing):
|
|
containers = self.containers.list()
|
|
cont_data = [c._apidict for c in containers]
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.get_account.return_value = [{}, cont_data]
|
|
|
|
(conts, more) = api.swift.swift_get_containers(self.request)
|
|
|
|
self.assertEqual(len(containers), len(conts))
|
|
self.assertFalse(more)
|
|
swift_api.get_account.assert_called_once_with(
|
|
limit=1001, marker=None, prefix=None,
|
|
full_listing=full_listing)
|
|
|
|
def test_swift_get_containers_default(self, mock_swiftclient):
|
|
self._test_swift_get_containers(mock_swiftclient, full_listing=True)
|
|
|
|
@override_settings(SWIFT_PANEL_FULL_LISTING=False)
|
|
def test_swift_get_containers_full_list_false(self, mock_swiftclient):
|
|
self._test_swift_get_containers(mock_swiftclient, full_listing=False)
|
|
|
|
def test_swift_get_container_with_data(self, mock_swiftclient):
|
|
container = self.containers.first()
|
|
objects = self.objects.list()
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.get_object.return_value = (container, objects)
|
|
|
|
cont = api.swift.swift_get_container(self.request, container.name,
|
|
with_data=True)
|
|
|
|
self.assertEqual(container.name, cont.name)
|
|
self.assertEqual(len(objects), len(cont.data))
|
|
swift_api.get_object.assert_called_once_with(container.name, "")
|
|
|
|
def test_swift_get_container_without_data(self, mock_swiftclient):
|
|
container = self.containers.first()
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.head_container.return_value = container
|
|
|
|
cont = api.swift.swift_get_container(self.request,
|
|
container.name,
|
|
with_data=False)
|
|
|
|
self.assertEqual(cont.name, container.name)
|
|
self.assertIsNone(cont.data)
|
|
swift_api.head_container.assert_called_once_with(container.name)
|
|
|
|
def test_swift_create_duplicate_container(self, mock_swiftclient):
|
|
metadata = {'is_public': False}
|
|
container = self.containers.first()
|
|
headers = api.swift._metadata_to_header(metadata=(metadata))
|
|
swift_api = mock_swiftclient.return_value
|
|
# Check for existence, then create
|
|
swift_api.head_container.side_effect = self.exceptions.swift
|
|
swift_api.put_container.return_value = container
|
|
|
|
api.swift.swift_create_container(self.request,
|
|
container.name,
|
|
metadata=(metadata))
|
|
|
|
swift_api.head_container.assert_called_once_with(container.name)
|
|
swift_api.put_container.assert_called_once_with(container.name,
|
|
headers=headers)
|
|
|
|
def test_swift_create_container(self, mock_swiftclient):
|
|
metadata = {'is_public': True}
|
|
container = self.containers.first()
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.head_container.return_value = container
|
|
|
|
with self.assertRaises(exceptions.AlreadyExists):
|
|
api.swift.swift_create_container(self.request,
|
|
container.name,
|
|
metadata=(metadata))
|
|
|
|
swift_api.head_container.assert_called_once_with(container.name)
|
|
|
|
def test_swift_create_container_with_storage_policy(self, mock_swiftclient):
|
|
metadata = {'is_public': True, 'storage_policy': 'nz-o1-mr-r3'}
|
|
container = self.containers.first()
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.head_container.return_value = container
|
|
|
|
headers = api.swift._metadata_to_header(metadata=(metadata))
|
|
self.assertEqual(headers["x-storage-policy"], 'nz-o1-mr-r3')
|
|
|
|
with self.assertRaises(exceptions.AlreadyExists):
|
|
api.swift.swift_create_container(self.request,
|
|
container.name,
|
|
metadata=(metadata))
|
|
|
|
swift_api.head_container.assert_called_once_with(container.name)
|
|
|
|
def test_metadata_to_headers(self, mock_swiftclient):
|
|
metadata = {'is_public': True, 'storage_policy': 'nz-o1-mr-r3'}
|
|
headers = api.swift._metadata_to_header(metadata=(metadata))
|
|
self.assertEqual(headers["x-storage-policy"], 'nz-o1-mr-r3')
|
|
self.assertEqual(headers["x-container-read"], '.r:*,.rlistings')
|
|
|
|
def test_metadata_to_headers_without_metadata(self, mock_swiftclient):
|
|
metadata = {}
|
|
headers = api.swift._metadata_to_header(metadata=(metadata))
|
|
self.assertNotIn("x-storage-policy", headers)
|
|
self.assertNotIn("x-container-read", headers)
|
|
|
|
def test_swift_update_container(self, mock_swiftclient):
|
|
metadata = {'is_public': True}
|
|
container = self.containers.first()
|
|
swift_api = mock_swiftclient.return_value
|
|
headers = api.swift._metadata_to_header(metadata=(metadata))
|
|
swift_api.post_container.return_value = container
|
|
|
|
api.swift.swift_update_container(self.request,
|
|
container.name,
|
|
metadata=(metadata))
|
|
|
|
swift_api.post_container.assert_called_once_with(container.name,
|
|
headers=headers)
|
|
|
|
def _test_swift_get_objects(self, mock_swiftclient, full_listing):
|
|
container = self.containers.first()
|
|
objects = self.objects.list()
|
|
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.get_container.return_value = [{}, objects]
|
|
|
|
(objs, more) = api.swift.swift_get_objects(self.request,
|
|
container.name)
|
|
|
|
self.assertEqual(len(objects), len(objs))
|
|
self.assertFalse(more)
|
|
swift_api.get_container.assert_called_once_with(
|
|
container.name,
|
|
limit=1001,
|
|
marker=None,
|
|
prefix=None,
|
|
delimiter='/',
|
|
full_listing=full_listing)
|
|
|
|
def test_swift_get_objects_default(self, mock_swiftclient):
|
|
self._test_swift_get_objects(mock_swiftclient, full_listing=True)
|
|
|
|
@override_settings(SWIFT_PANEL_FULL_LISTING=False)
|
|
def test_swift_get_objects_full_list_false(self, mock_swiftclient):
|
|
self._test_swift_get_objects(mock_swiftclient, full_listing=False)
|
|
|
|
def test_swift_get_object_with_data_non_chunked(self, mock_swiftclient):
|
|
container = self.containers.first()
|
|
object = self.objects.first()
|
|
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.get_object.return_value = [object, object.data]
|
|
|
|
obj = api.swift.swift_get_object(self.request, container.name,
|
|
object.name, resp_chunk_size=None)
|
|
|
|
self.assertEqual(object.name, obj.name)
|
|
swift_api.get_object.assert_called_once_with(
|
|
container.name, object.name, resp_chunk_size=None)
|
|
|
|
def test_swift_get_object_with_data_chunked(self, mock_swiftclient):
|
|
container = self.containers.first()
|
|
object = self.objects.first()
|
|
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.get_object.return_value = [object, object.data]
|
|
|
|
obj = api.swift.swift_get_object(
|
|
self.request, container.name, object.name)
|
|
|
|
self.assertEqual(object.name, obj.name)
|
|
swift_api.get_object.assert_called_once_with(
|
|
container.name, object.name, resp_chunk_size=api.swift.CHUNK_SIZE)
|
|
|
|
def test_swift_get_object_without_data(self, mock_swiftclient):
|
|
container = self.containers.first()
|
|
object = self.objects.first()
|
|
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.head_object.return_value = object
|
|
|
|
obj = api.swift.swift_get_object(self.request,
|
|
container.name,
|
|
object.name,
|
|
with_data=False)
|
|
|
|
self.assertEqual(object.name, obj.name)
|
|
self.assertIsNone(obj.data)
|
|
swift_api.head_object.assert_called_once_with(container.name,
|
|
object.name)
|
|
|
|
def test_swift_create_pseudo_folder(self, mock_swiftclient):
|
|
container = self.containers.first()
|
|
folder = self.folder.first()
|
|
swift_api = mock_swiftclient.return_value
|
|
exc = self.exceptions.swift
|
|
swift_api.head_object.side_effect = exc
|
|
swift_api.put_object.return_value = folder
|
|
|
|
api.swift.swift_create_pseudo_folder(self.request,
|
|
container.name,
|
|
folder.name)
|
|
|
|
swift_api.head_object.assert_called_once_with(container.name,
|
|
folder.name)
|
|
swift_api.put_object.assert_called_once_with(container.name,
|
|
folder.name,
|
|
None,
|
|
headers={})
|
|
|
|
def test_swift_create_duplicate_folder(self, mock_swiftclient):
|
|
container = self.containers.first()
|
|
folder = self.folder.first()
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.head_object.return_value = folder
|
|
|
|
with self.assertRaises(exceptions.AlreadyExists):
|
|
api.swift.swift_create_pseudo_folder(self.request,
|
|
container.name,
|
|
folder.name)
|
|
|
|
swift_api.head_object.assert_called_once_with(container.name,
|
|
folder.name)
|
|
|
|
def test_swift_upload_object(self, mock_swiftclient):
|
|
container = self.containers.first()
|
|
obj = self.objects.first()
|
|
fake_name = 'fake_object.jpg'
|
|
|
|
class FakeFile(object):
|
|
def __init__(self):
|
|
self.name = fake_name
|
|
self.data = obj.data
|
|
self.size = len(obj.data)
|
|
|
|
headers = {'X-Object-Meta-Orig-Filename': fake_name}
|
|
|
|
swift_api = mock_swiftclient.return_value
|
|
test_file = FakeFile()
|
|
swift_api.put_object.return_value = None
|
|
|
|
api.swift.swift_upload_object(self.request,
|
|
container.name,
|
|
obj.name,
|
|
test_file)
|
|
|
|
swift_api.put_object.assert_called_once_with(
|
|
container.name,
|
|
obj.name,
|
|
test.IsA(FakeFile),
|
|
content_length=test_file.size,
|
|
headers=headers)
|
|
|
|
def test_swift_upload_object_without_file(self, mock_swiftclient):
|
|
container = self.containers.first()
|
|
obj = self.objects.first()
|
|
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.put_object.return_value = None
|
|
|
|
response = api.swift.swift_upload_object(self.request,
|
|
container.name,
|
|
obj.name,
|
|
None)
|
|
|
|
self.assertEqual(0, response['bytes'])
|
|
swift_api.put_object.assert_called_once_with(
|
|
container.name,
|
|
obj.name,
|
|
None,
|
|
content_length=0,
|
|
headers={})
|
|
|
|
def test_swift_object_exists(self, mock_swiftclient):
|
|
container = self.containers.first()
|
|
obj = self.objects.first()
|
|
|
|
swift_api = mock_swiftclient.return_value
|
|
swift_api.head_object.side_effect = [container, self.exceptions.swift]
|
|
|
|
args = self.request, container.name, obj.name
|
|
|
|
self.assertTrue(api.swift.swift_object_exists(*args))
|
|
# Again, for a "non-existent" object
|
|
self.assertFalse(api.swift.swift_object_exists(*args))
|
|
|
|
self.assertEqual(2, swift_api.head_object.call_count)
|
|
swift_api.head_object.assert_has_calls([
|
|
mock.call(container.name, obj.name),
|
|
mock.call(container.name, obj.name),
|
|
])
|