openstack/deb-ceilometer
Code Issues Proposed changes

Switch to use Glance v2 in image pollsters

Browse Source 
Since the Glance v1 APIs won't be maintained any more, it is better to
switch use Glance v2 for in image pollsters.

Change-Id: Ib2df3bb4fdd12649bddffd624714707e1369f6af
This commit is contained in:
liusheng 2016-08-01 15:35:14 +08:00
parent d191741cdd
commit f8933f4abd
7 changed files with 158 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
ceilometer/image/discovery.py Normal file
View File

@ -0,0 +1,43 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import glanceclient
from oslo_config import cfg
from ceilometer.agent import plugin_base
from ceilometer import keystone_client
SERVICE_OPTS = [
cfg.StrOpt('glance',
default='image',
help='Glance service type.'),
]
cfg.CONF.register_opts(SERVICE_OPTS, group='service_types')
cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client')
class ImagesDiscovery(plugin_base.DiscoveryBase):
def __init__(self):
super(ImagesDiscovery, self).__init__()
conf = cfg.CONF.service_credentials
self.glance_client = glanceclient.Client(
version='2',
session=keystone_client.get_session(),
region_name=conf.region_name,
interface=conf.interface,
service_type=cfg.CONF.service_types.glance)
def discover(self, manager, param=None):
"""Discover resources to monitor."""
return self.glance_client.images.list()

65
ceilometer/image/glance.py
View File

@ -17,63 +17,14 @@
from __future__ import absolute_import from __future__ import absolute_import
import glanceclient
from oslo_config import cfg
from ceilometer.agent import plugin_base from ceilometer.agent import plugin_base
from ceilometer import keystone_client
from ceilometer import sample from ceilometer import sample
OPTS = [
cfg.IntOpt('glance_page_size',
default=0,
help="Number of items to request in "
"each paginated Glance API request "
"(parameter used by glanceclient). "
"If this is less than or equal to 0, "
"page size is not specified "
"(default value in glanceclient is used)."),
]
SERVICE_OPTS = [
cfg.StrOpt('glance',
default='image',
help='Glance service type.'),
]
cfg.CONF.register_opts(OPTS)
cfg.CONF.register_opts(SERVICE_OPTS, group='service_types')
class _Base(plugin_base.PollsterBase): class _Base(plugin_base.PollsterBase):
@property @property
def default_discovery(self): def default_discovery(self):
return 'endpoint:%s' % cfg.CONF.service_types.glance return 'images'
@staticmethod
def get_glance_client(ksclient, endpoint):
# hard-code v1 glance API version selection while v2 API matures
return glanceclient.Client('1',
session=keystone_client.get_session(),
endpoint=endpoint,
auth=ksclient.session.auth)
def _get_images(self, ksclient, endpoint):
client = self.get_glance_client(ksclient, endpoint)
page_size = cfg.CONF.glance_page_size
kwargs = {}
if page_size > 0:
kwargs['page_size'] = page_size
return client.images.list(filters={"is_public": None}, **kwargs)
def _iter_images(self, ksclient, cache, endpoint):
"""Iterate over all images."""
key = '%s-images' % endpoint
if key not in cache:
cache[key] = list(self._get_images(ksclient, endpoint))
return iter(cache[key])
@staticmethod @staticmethod
def extract_image_metadata(image): def extract_image_metadata(image):
@ -81,26 +32,23 @@ class _Base(plugin_base.PollsterBase):
for k in for k in
[ [
"status", "status",
"is_public", "visibility",
"name", "name",
"deleted",
"container_format", "container_format",
"created_at", "created_at",
"disk_format", "disk_format",
"updated_at", "updated_at",
"properties",
"min_disk", "min_disk",
"protected", "protected",
"checksum", "checksum",
"deleted_at",
"min_ram", "min_ram",
"size", ]) "tags",
"virtual_size"])
class ImagePollster(_Base): class ImagePollster(_Base):
def get_samples(self, manager, cache, resources): def get_samples(self, manager, cache, resources):
for endpoint in resources: for image in resources:
for image in self._iter_images(manager.keystone, cache, endpoint):
yield sample.Sample( yield sample.Sample(
name='image', name='image',
type=sample.TYPE_GAUGE, type=sample.TYPE_GAUGE,
@ -115,8 +63,7 @@ class ImagePollster(_Base):
class ImageSizePollster(_Base): class ImageSizePollster(_Base):
def get_samples(self, manager, cache, resources): def get_samples(self, manager, cache, resources):
for endpoint in resources: for image in resources:
for image in self._iter_images(manager.keystone, cache, endpoint):
yield sample.Sample( yield sample.Sample(
name='image.size', name='image.size',
type=sample.TYPE_GAUGE, type=sample.TYPE_GAUGE,

2
ceilometer/nova_client.py
View File

@ -40,7 +40,7 @@ SERVICE_OPTS = [
cfg.CONF.register_opts(OPTS) cfg.CONF.register_opts(OPTS)
cfg.CONF.register_opts(SERVICE_OPTS, group='service_types') cfg.CONF.register_opts(SERVICE_OPTS, group='service_types')
cfg.CONF.import_opt('http_timeout', 'ceilometer.service') cfg.CONF.import_opt('http_timeout', 'ceilometer.service')
cfg.CONF.import_opt('glance', 'ceilometer.image.glance', 'service_types') cfg.CONF.import_opt('glance', 'ceilometer.image.discovery', 'service_types')
cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client') cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client')
LOG = log.getLogger(__name__) LOG = log.getLogger(__name__)

5
ceilometer/opts.py
View File

@ -34,7 +34,7 @@ import ceilometer.dispatcher.gnocchi
import ceilometer.energy.kwapi import ceilometer.energy.kwapi
import ceilometer.event.converter import ceilometer.event.converter
import ceilometer.hardware.discovery import ceilometer.hardware.discovery
import ceilometer.image.glance import ceilometer.image.discovery
import ceilometer.ipmi.notifications.ironic import ceilometer.ipmi.notifications.ironic
import ceilometer.ipmi.platform.intel_node_manager import ceilometer.ipmi.platform.intel_node_manager
import ceilometer.ipmi.pollsters import ceilometer.ipmi.pollsters
@ -67,7 +67,6 @@ def list_opts():
ceilometer.compute.virt.inspector.OPTS, ceilometer.compute.virt.inspector.OPTS,
ceilometer.compute.virt.libvirt.inspector.OPTS, ceilometer.compute.virt.libvirt.inspector.OPTS,
ceilometer.dispatcher.OPTS, ceilometer.dispatcher.OPTS,
ceilometer.image.glance.OPTS,
ceilometer.ipmi.notifications.ironic.OPTS, ceilometer.ipmi.notifications.ironic.OPTS,
ceilometer.middleware.OPTS, ceilometer.middleware.OPTS,
ceilometer.network.notifications.OPTS, ceilometer.network.notifications.OPTS,
@ -113,7 +112,7 @@ def list_opts():
loading.get_auth_plugin_conf_options('password'))), loading.get_auth_plugin_conf_options('password'))),
('service_types', ('service_types',
itertools.chain(ceilometer.energy.kwapi.SERVICE_OPTS, itertools.chain(ceilometer.energy.kwapi.SERVICE_OPTS,
ceilometer.image.glance.SERVICE_OPTS, ceilometer.image.discovery.SERVICE_OPTS,
ceilometer.neutron_client.SERVICE_OPTS, ceilometer.neutron_client.SERVICE_OPTS,
ceilometer.nova_client.SERVICE_OPTS, ceilometer.nova_client.SERVICE_OPTS,
ceilometer.objectstore.rgw.SERVICE_OPTS, ceilometer.objectstore.rgw.SERVICE_OPTS,

261
ceilometer/tests/unit/image/test_glance.py
View File

@ -14,214 +14,107 @@
# under the License. # under the License.
import mock import mock
from oslo_config import fixture as fixture_config
from oslotest import base
from oslotest import mockpatch
from ceilometer.agent import manager from ceilometer.agent import manager
from ceilometer.image import glance from ceilometer.image import glance
import ceilometer.tests.base as base
IMAGE_LIST = [ IMAGE_LIST = [
type('Image', (object,), type('Image', (object,),
{u'status': u'queued', {u'status': u'active',
u'name': "some name", u'tags': [],
u'deleted': False, u'kernel_id': u'fd24d91a-dfd5-4a3c-b990-d4563eb27396',
u'container_format': None, u'container_format': u'ami',
u'created_at': u'2012-09-18T16:29:46',
u'disk_format': None,
u'updated_at': u'2012-09-18T16:29:46',
u'properties': {},
u'min_disk': 0,
u'protected': False,
u'id': u'1d21a8d0-25f4-4e0a-b4ec-85f40237676b',
u'location': None,
u'checksum': None,
u'owner': u'4c8364fc20184ed7971b76602aa96184',
u'is_public': True,
u'deleted_at': None,
u'min_ram': 0, u'min_ram': 0,
u'size': 2048}), u'ramdisk_id': u'd629522b-ebaa-4c92-9514-9e31fe760d18',
u'updated_at': u'2016-06-20T13: 34: 41Z',
u'visibility': u'public',
u'owner': u'6824974c08974d4db864bbaa6bc08303',
u'file': u'/v2/images/fda54a44-3f96-40bf-ab07-0a4ce9e1761d/file',
u'min_disk': 0,
u'virtual_size': None,
u'id': u'fda54a44-3f96-40bf-ab07-0a4ce9e1761d',
u'size': 25165824,
u'name': u'cirros-0.3.4-x86_64-uec',
u'checksum': u'eb9139e4942121f22bbc2afc0400b2a4',
u'created_at': u'2016-06-20T13: 34: 40Z',
u'disk_format': u'ami',
u'protected': False,
u'schema': u'/v2/schemas/image'}),
type('Image', (object,), type('Image', (object,),
{u'status': u'active', {u'status': u'active',
u'name': "hello world", u'tags': [],
u'deleted': False, u'container_format': u'ari',
u'container_format': None,
u'created_at': u'2012-09-18T16:27:41',
u'disk_format': None,
u'updated_at': u'2012-09-18T16:27:41',
u'properties': {},
u'min_disk': 0,
u'protected': False,
u'id': u'22be9f90-864d-494c-aa74-8035fd535989',
u'location': None,
u'checksum': None,
u'owner': u'9e4f98287a0246daa42eaf4025db99d4',
u'is_public': True,
u'deleted_at': None,
u'min_ram': 0, u'min_ram': 0,
u'size': 0}), u'updated_at': u'2016-06-20T13: 34: 38Z',
u'visibility': u'public',
u'owner': u'6824974c08974d4db864bbaa6bc08303',
u'file': u'/v2/images/d629522b-ebaa-4c92-9514-9e31fe760d18/file',
u'min_disk': 0,
u'virtual_size': None,
u'id': u'd629522b-ebaa-4c92-9514-9e31fe760d18',
u'size': 3740163,
u'name': u'cirros-0.3.4-x86_64-uec-ramdisk',
u'checksum': u'be575a2b939972276ef675752936977f',
u'created_at': u'2016-06-20T13: 34: 37Z',
u'disk_format': u'ari',
u'protected': False,
u'schema': u'/v2/schemas/image'}),
type('Image', (object,), type('Image', (object,),
{u'status': u'queued', {u'status': u'active',
u'name': None, u'tags': [],
u'deleted': False, u'container_format': u'aki',
u'container_format': None,
u'created_at': u'2012-09-18T16:23:27',
u'disk_format': "raw",
u'updated_at': u'2012-09-18T16:23:27',
u'properties': {},
u'min_disk': 0,
u'protected': False,
u'id': u'8d133f6c-38a8-403c-b02c-7071b69b432d',
u'location': None,
u'checksum': None,
u'owner': u'5f8806a76aa34ee8b8fc8397bd154319',
u'is_public': True,
u'deleted_at': None,
u'min_ram': 0, u'min_ram': 0,
u'size': 1024}), u'updated_at': u'2016-06-20T13: 34: 35Z',
type('Image', (object,), u'visibility': u'public',
{u'status': u'queued', u'owner': u'6824974c08974d4db864bbaa6bc08303',
u'name': "some name", u'file': u'/v2/images/fd24d91a-dfd5-4a3c-b990-d4563eb27396/file',
u'deleted': False,
u'container_format': None,
u'created_at': u'2012-09-18T16:29:46',
u'disk_format': None,
u'updated_at': u'2012-09-18T16:29:46',
u'properties': {},
u'min_disk': 0, u'min_disk': 0,
u'virtual_size': None,
u'id': u'fd24d91a-dfd5-4a3c-b990-d4563eb27396',
u'size': 4979632,
u'name': u'cirros-0.3.4-x86_64-uec-kernel',
u'checksum': u'8a40c862b5735975d82605c1dd395796',
u'created_at': u'2016-06-20T13: 34: 35Z',
u'disk_format': u'aki',
u'protected': False, u'protected': False,
u'id': u'e753b196-49b4-48e8-8ca5-09ebd9805f40', u'schema': u'/v2/schemas/image'}),
u'location': None,
u'checksum': None,
u'owner': u'4c8364fc20184ed7971b76602aa96184',
u'is_public': True,
u'deleted_at': None,
u'min_ram': 0,
u'size': 2048}),
] ]
ENDPOINT = 'end://point'
class _BaseObject(object):
pass
class FakeGlanceClient(object):
class images(object):
pass
class TestManager(manager.AgentManager):
def __init__(self):
super(TestManager, self).__init__()
self._keystone = mock.Mock()
access = self._keystone.session.auth.get_access.return_value
access.service_catalog.get_endpoints = mock.Mock(
return_value={'image': mock.ANY})
class TestImagePollsterPageSize(base.BaseTestCase): class TestImagePollsterPageSize(base.BaseTestCase):
@staticmethod
def fake_get_glance_client(ksclient, endpoint):
glanceclient = FakeGlanceClient()
glanceclient.images.list = mock.MagicMock(return_value=IMAGE_LIST)
return glanceclient
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock()) @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def setUp(self): def setUp(self):
super(TestImagePollsterPageSize, self).setUp() super(TestImagePollsterPageSize, self).setUp()
self.manager = TestManager() self.manager = manager.AgentManager()
self.useFixture(mockpatch.PatchObject( self.pollster = glance.ImageSizePollster()
glance._Base, 'get_glance_client',
side_effect=self.fake_get_glance_client))
self.CONF = self.useFixture(fixture_config.Config()).conf
def _do_test_iter_images(self, page_size=0, length=0): def test_image_pollster(self):
self.CONF.set_override("glance_page_size", page_size) image_samples = list(
images = list(glance.ImagePollster(). self.pollster.get_samples(self.manager, {}, resources=IMAGE_LIST))
_iter_images(self.manager.keystone, {}, ENDPOINT)) self.assertEqual(3, len(image_samples))
kwargs = {} self.assertEqual('image.size', image_samples[0].name)
if page_size > 0: self.assertEqual(25165824, image_samples[0].volume)
kwargs['page_size'] = page_size self.assertEqual('6824974c08974d4db864bbaa6bc08303',
FakeGlanceClient.images.list.assert_called_with( image_samples[0].project_id)
filters={'is_public': None}, **kwargs) self.assertEqual('fda54a44-3f96-40bf-ab07-0a4ce9e1761d',
self.assertEqual(length, len(images)) image_samples[0].resource_id)
def test_page_size(self):
self._do_test_iter_images(100, 4)
def test_page_size_default(self):
self._do_test_iter_images(length=4)
def test_page_size_negative_number(self):
self._do_test_iter_images(-1, 4)
class TestImagePollster(base.BaseTestCase): class TestImagePageSize(base.BaseTestCase):
@staticmethod
def fake_get_glance_client(ksclient, endpoint):
glanceclient = _BaseObject()
setattr(glanceclient, "images", _BaseObject())
setattr(glanceclient.images,
"list", lambda *args, **kwargs: iter(IMAGE_LIST))
return glanceclient
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock()) @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def setUp(self): def setUp(self):
super(TestImagePollster, self).setUp() super(TestImagePageSize, self).setUp()
self.manager = TestManager() self.manager = manager.AgentManager()
self.useFixture(mockpatch.PatchObject( self.pollster = glance.ImagePollster()
glance._Base, 'get_glance_client',
side_effect=self.fake_get_glance_client))
def test_default_discovery(self): def test_image_pollster(self):
pollster = glance.ImagePollster() image_samples = list(
self.assertEqual('endpoint:image', pollster.default_discovery) self.pollster.get_samples(self.manager, {}, resources=IMAGE_LIST))
self.assertEqual(3, len(image_samples))
def test_iter_images(self): self.assertEqual('image', image_samples[0].name)
# Tests whether the iter_images method returns a unique image self.assertEqual(1, image_samples[0].volume)
# list when there is nothing in the cache self.assertEqual('6824974c08974d4db864bbaa6bc08303',
images = list(glance.ImagePollster(). image_samples[0].project_id)
_iter_images(self.manager.keystone, {}, ENDPOINT)) self.assertEqual('fda54a44-3f96-40bf-ab07-0a4ce9e1761d',
self.assertEqual(len(set(image.id for image in images)), len(images)) image_samples[0].resource_id)
def test_iter_images_cached(self):
# Tests whether the iter_images method returns the values from
# the cache
cache = {'%s-images' % ENDPOINT: []}
images = list(glance.ImagePollster().
_iter_images(self.manager.keystone, cache,
ENDPOINT))
self.assertEqual([], images)
def test_image(self):
samples = list(glance.ImagePollster().get_samples(self.manager, {},
[ENDPOINT]))
self.assertEqual(4, len(samples))
for sample in samples:
self.assertEqual(1, sample.volume)
def test_image_size(self):
samples = list(glance.ImageSizePollster().get_samples(self.manager,
{},
[ENDPOINT]))
self.assertEqual(4, len(samples))
for image in IMAGE_LIST:
self.assertTrue(
any(map(lambda sample: sample.volume == image.size,
samples)))
def test_image_get_sample_names(self):
samples = list(glance.ImagePollster().get_samples(self.manager, {},
[ENDPOINT]))
self.assertEqual(set(['image']), set([s.name for s in samples]))
def test_image_size_get_sample_names(self):
samples = list(glance.ImageSizePollster().get_samples(self.manager,
{},
[ENDPOINT]))
self.assertEqual(set(['image.size']), set([s.name for s in samples]))

8
releasenotes/notes/use-glance-v2-in-image-pollsters-137a315577d5dc4c.yaml Normal file
View File

@ -0,0 +1,8 @@
---
features:
- Since the Glance v1 APIs won't be maintained any more, this change
add the support of glance v2 in images pollsters.
upgrade:
- >
The option `glance_page_size' has been removed because it's not actually needed.

1
setup.cfg
View File

@ -75,6 +75,7 @@ ceilometer.discover.central =
fw_policy = ceilometer.network.services.discovery:FirewallPolicyDiscovery fw_policy = ceilometer.network.services.discovery:FirewallPolicyDiscovery
tripleo_overcloud_nodes = ceilometer.hardware.discovery:NodesDiscoveryTripleO tripleo_overcloud_nodes = ceilometer.hardware.discovery:NodesDiscoveryTripleO
fip_services = ceilometer.network.services.discovery:FloatingIPDiscovery fip_services = ceilometer.network.services.discovery:FloatingIPDiscovery
images = ceilometer.image.discovery:ImagesDiscovery
ceilometer.discover.ipmi = ceilometer.discover.ipmi =
local_node = ceilometer.agent.discovery.localnode:LocalNodeDiscovery local_node = ceilometer.agent.discovery.localnode:LocalNodeDiscovery