From c6694dea349f4aad63cc06fb42ae5c5907e902ed Mon Sep 17 00:00:00 2001 From: Rikimaru Honjo Date: Mon, 11 Aug 2014 22:00:50 +0900 Subject: [PATCH] Set page size when Glance API request is called The glance related pollsters in central agent uses glanceclient to get image-list from glance. Since glanceclient has pagination feature, it calls glance API for every page sizes. When glanceclient is used by Central Agent, default value of page size, that is 20, is used, since Central Agent doesn't specify page size. However, default value of page size is too small when there is a large number of images. Therefore we need to specify page size from Central Agent. Change-Id: I5f6457234a2963bbe567d078f2c8bb098c8a311c Closes-bug: #1340577 --- ceilometer/image/glance.py | 24 ++++++++++++--- ceilometer/tests/image/test_glance.py | 44 +++++++++++++++++++++++++++ doc/source/configuration.rst | 6 ++++ 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/ceilometer/image/glance.py b/ceilometer/image/glance.py index e102f65ce..85bf6851b 100644 --- a/ceilometer/image/glance.py +++ b/ceilometer/image/glance.py @@ -28,6 +28,20 @@ from ceilometer.openstack.common import timeutils from ceilometer import sample +OPTS = [ + cfg.IntOpt('glance_page_size', + default=0, + help="Number of items to request in " + "each paginated Glance API request " + "(parameter used by glancecelient). " + "If this is less than or equal to 0, " + "page size is not specified " + "(default value in glanceclient is used)."), +] + +cfg.CONF.register_opts(OPTS) + + class _Base(plugin.CentralPollster): @staticmethod @@ -45,13 +59,15 @@ class _Base(plugin.CentralPollster): def _get_images(self, ksclient): client = self.get_glance_client(ksclient) - # TODO(eglynn): use pagination to protect against unbounded - # memory usage + page_size = cfg.CONF.glance_page_size + kwargs = {} + if page_size > 0: + kwargs['page_size'] = page_size rawImageList = list(itertools.chain( - client.images.list(filters={"is_public": True}), + client.images.list(filters={"is_public": True}, **kwargs), # TODO(eglynn): extend glance API with all_tenants logic to # avoid second call to retrieve private images - client.images.list(filters={"is_public": False}))) + client.images.list(filters={"is_public": False}, **kwargs))) # When retrieving images from glance, glance will check # whether the user is of 'admin_role' which is diff --git a/ceilometer/tests/image/test_glance.py b/ceilometer/tests/image/test_glance.py index 65a61709e..37dbd6819 100644 --- a/ceilometer/tests/image/test_glance.py +++ b/ceilometer/tests/image/test_glance.py @@ -20,6 +20,7 @@ import mock from ceilometer.central import manager from ceilometer.image import glance from ceilometer.openstack.common import context +from ceilometer.openstack.common.fixture import config from ceilometer.openstack.common.fixture import mockpatch from ceilometer.openstack.common import test @@ -108,6 +109,11 @@ class _BaseObject(object): pass +class FakeGlanceClient(object): + class images(object): + pass + + class TestManager(manager.AgentManager): def __init__(self): @@ -115,6 +121,44 @@ class TestManager(manager.AgentManager): self.keystone = mock.Mock() +class TestImagePollsterPageSize(test.BaseTestCase): + + def fake_get_glance_client(self, ksclient): + glanceclient = FakeGlanceClient() + glanceclient.images.list = mock.MagicMock(return_value=IMAGE_LIST) + return glanceclient + + @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock()) + def setUp(self): + super(TestImagePollsterPageSize, self).setUp() + self.context = context.get_admin_context() + self.manager = TestManager() + self.useFixture(mockpatch.PatchObject( + glance._Base, 'get_glance_client', + side_effect=self.fake_get_glance_client)) + self.CONF = self.useFixture(config.Config()).conf + + def _do_test_iter_images(self, page_size=0): + self.CONF.set_override("glance_page_size", page_size) + images = list(glance.ImagePollster(). + _iter_images(self.manager.keystone, {})) + kwargs = {} + if page_size > 0: + kwargs['page_size'] = page_size + FakeGlanceClient.images.list.assert_called_with( + filters={'is_public': False}, **kwargs) + self.assertEqual(len(set(image.id for image in images)), len(images)) + + def test_page_size(self): + self._do_test_iter_images(100) + + def test_page_size_default(self): + self._do_test_iter_images() + + def test_page_size_negative_number(self): + self._do_test_iter_images(-1) + + class TestImagePollster(test.BaseTestCase): def fake_get_glance_client(self, ksclient): diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 674c1e1fa..ae9d25db8 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -43,6 +43,12 @@ database_connection mongodb://localhost:27017/ceilometer Database metering_api_port 8777 The port for the ceilometer API server reseller_prefix AUTH\_ Prefix used by swift for reseller token nova_http_log_debug False Log request/response parameters between nova and ceilometer +glance_page_size 0 Number of items to request in each paginated Glance API + request (parameter used by glancecelient). If this is less + than or equal to 0, page size is not specified (default value + in glanceclient is used). It is better to check and set + appropriate value in line with each environment when calling + glanceclient, than to define higher default value. =============================== ==================================== ============================================================== Service polling authentication