openstack/ceilometer
Code Issues Proposed changes

libvirt: rewrite the error handling

Browse Source 
Error handling of libvirt driver work 'sometimes'.
Some error are catched only with we lookup for a instance
uuid, but not for other libvirt operation.

This change rewrites the logic to catch libvirt error on
each inpector method. This ensures we catch all errors whatever
which libvirt method raises it.

We also leverage tenacity instead of the custom retry code.

Change-Id: Idd54c18ece42c2dce3baf82626d30d5c2e5a49d6
This commit is contained in:
Mehdi Abaakouk 2016-11-16 12:23:01 +01:00
parent 30fec85a03
commit 66179f89a0
8 changed files with 283 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ceilometer/compute/discovery.py
View File

@ -107,7 +107,6 @@ class InstanceDiscovery(plugin_base.DiscoveryBase):
self.expiration_time = conf.compute.resource_update_interval self.expiration_time = conf.compute.resource_update_interval
self.cache_expiry = conf.compute.resource_cache_expiry self.cache_expiry = conf.compute.resource_cache_expiry
if self.method == "libvirt_metadata": if self.method == "libvirt_metadata":
self._connection = None
# 4096 instances on a compute should be enough :) # 4096 instances on a compute should be enough :)
self._flavor_cache = cachetools.LRUCache(4096) self._flavor_cache = cachetools.LRUCache(4096)
else: else:
@ -117,9 +116,7 @@ class InstanceDiscovery(plugin_base.DiscoveryBase):
@property @property
def connection(self): def connection(self):
if not self._connection: return libvirt_utils.refresh_libvirt_connection(self.conf, self)
self._connection = libvirt_utils.get_libvirt_connection(self.conf)
return self._connection
def discover(self, manager, param=None): def discover(self, manager, param=None):
"""Discover resources to monitor.""" """Discover resources to monitor."""

15
ceilometer/compute/pollsters/__init__.py
View File

@ -25,13 +25,18 @@ from ceilometer.compute.virt import inspector as virt_inspector
@six.add_metaclass(abc.ABCMeta) @six.add_metaclass(abc.ABCMeta)
class BaseComputePollster(plugin_base.PollsterBase): class BaseComputePollster(plugin_base.PollsterBase):
@property def setup_environment(self):
def inspector(self): super(BaseComputePollster, self).setup_environment()
self.inspector = self._get_inspector(self.conf)
@classmethod
def _get_inspector(cls, conf):
# FIXME(sileht): This doesn't looks threadsafe...
try: try:
inspector = self._inspector inspector = cls._inspector
except AttributeError: except AttributeError:
inspector = virt_inspector.get_hypervisor_inspector(self.conf) inspector = virt_inspector.get_hypervisor_inspector(conf)
BaseComputePollster._inspector = inspector cls._inspector = inspector
return inspector return inspector
@property @property

159
ceilometer/compute/virt/libvirt/inspector.py
View File

@ -27,7 +27,7 @@ except ImportError:
from ceilometer.compute.pollsters import util from ceilometer.compute.pollsters import util
from ceilometer.compute.virt import inspector as virt_inspector from ceilometer.compute.virt import inspector as virt_inspector
from ceilometer.compute.virt.libvirt import utils as libvirt_utils from ceilometer.compute.virt.libvirt import utils as libvirt_utils
from ceilometer.i18n import _LW, _LE, _ from ceilometer.i18n import _LW, _
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@ -36,37 +36,32 @@ class LibvirtInspector(virt_inspector.Inspector):
def __init__(self, conf): def __init__(self, conf):
super(LibvirtInspector, self).__init__(conf) super(LibvirtInspector, self).__init__(conf)
self._connection = None # NOTE(sileht): create a connection on startup
self.connection
@property @property
def connection(self): def connection(self):
if not self._connection: return libvirt_utils.refresh_libvirt_connection(self.conf, self)
self._connection = libvirt_utils.get_libvirt_connection(self.conf)
return self._connection
@libvirt_utils.retry_on_disconnect
def _lookup_by_uuid(self, instance): def _lookup_by_uuid(self, instance):
instance_name = util.instance_name(instance) instance_name = util.instance_name(instance)
try: try:
return self.connection.lookupByUUIDString(instance.id) return self.connection.lookupByUUIDString(instance.id)
except Exception as ex: except libvirt.libvirtError as ex:
if not libvirt or not isinstance(ex, libvirt.libvirtError): if libvirt_utils.is_disconnection_exception(ex):
raise virt_inspector.InspectorException(six.text_type(ex))
error_code = ex.get_error_code()
if (error_code in (libvirt.VIR_ERR_SYSTEM_ERROR,
libvirt.VIR_ERR_INTERNAL_ERROR) and
ex.get_error_domain() in (libvirt.VIR_FROM_REMOTE,
libvirt.VIR_FROM_RPC)):
raise raise
msg = _("Error from libvirt while looking up instance " msg = _("Error from libvirt while looking up instance "
"<name=%(name)s, id=%(id)s>: " "<name=%(name)s, id=%(id)s>: "
"[Error Code %(error_code)s] " "[Error Code %(error_code)s] "
"%(ex)s") % {'name': instance_name, "%(ex)s") % {'name': instance_name,
'id': instance.id, 'id': instance.id,
'error_code': error_code, 'error_code': ex.get_error_code(),
'ex': ex} 'ex': ex}
raise virt_inspector.InstanceNotFoundException(msg) raise virt_inspector.InstanceNotFoundException(msg)
except Exception as ex:
raise virt_inspector.InspectorException(six.text_type(ex))
@libvirt_utils.retry_on_disconnect
def inspect_cpus(self, instance): def inspect_cpus(self, instance):
domain = self._get_domain_not_shut_off_or_raise(instance) domain = self._get_domain_not_shut_off_or_raise(instance)
# TODO(gordc): this can probably be cached since it can be used to get # TODO(gordc): this can probably be cached since it can be used to get
@ -76,31 +71,15 @@ class LibvirtInspector(virt_inspector.Inspector):
return virt_inspector.CPUStats(number=dom_stat['vcpu.current'], return virt_inspector.CPUStats(number=dom_stat['vcpu.current'],
time=dom_stat['cpu.time']) time=dom_stat['cpu.time'])
@libvirt_utils.raise_nodata_if_unsupported("l3 cache usage")
@libvirt_utils.retry_on_disconnect
def inspect_cpu_l3_cache(self, instance): def inspect_cpu_l3_cache(self, instance):
domain = self._lookup_by_uuid(instance) domain = self._lookup_by_uuid(instance)
try: stats = self.connection.domainListGetStats(
stats = self.connection.domainListGetStats( [domain], libvirt.VIR_DOMAIN_STATS_PERF)
[domain], libvirt.VIR_DOMAIN_STATS_PERF) perf = stats[0][1]
perf = stats[0][1] usage = perf["perf.cmt"]
usage = perf["perf.cmt"] return virt_inspector.CPUL3CacheUsageStats(l3_cache_usage=usage)
return virt_inspector.CPUL3CacheUsageStats(l3_cache_usage=usage)
except (KeyError, AttributeError) as e:
# NOTE(sileht): KeyError if for libvirt >=2.0.0,<2.3.0, the perf
# subsystem ws existing but not these attributes
# https://github.com/libvirt/libvirt/commit/bae660869de0612bee2a740083fb494c27e3f80c
msg = _('Perf is not supported by current version of libvirt, and '
'failed to inspect l3 cache usage of %(instance_uuid)s, '
'can not get info from libvirt: %(error)s') % {
'instance_uuid': instance.id, 'error': e}
raise virt_inspector.NoDataException(msg)
# domainListGetStats might launch an exception if the method or
# cmt perf event is not supported by the underlying hypervisor
# being used by libvirt.
except libvirt.libvirtError as e:
msg = _('Failed to inspect l3 cache usage of %(instance_uuid)s, '
'can not get info from libvirt: %(error)s') % {
'instance_uuid': instance.id, 'error': e}
raise virt_inspector.NoDataException(msg)
def _get_domain_not_shut_off_or_raise(self, instance): def _get_domain_not_shut_off_or_raise(self, instance):
instance_name = util.instance_name(instance) instance_name = util.instance_name(instance)
@ -116,6 +95,7 @@ class LibvirtInspector(virt_inspector.Inspector):
return domain return domain
@libvirt_utils.retry_on_disconnect
def inspect_vnics(self, instance): def inspect_vnics(self, instance):
domain = self._get_domain_not_shut_off_or_raise(instance) domain = self._get_domain_not_shut_off_or_raise(instance)
@ -150,6 +130,7 @@ class LibvirtInspector(virt_inspector.Inspector):
tx_errors=dom_stats[7]) tx_errors=dom_stats[7])
yield (interface, stats) yield (interface, stats)
@libvirt_utils.retry_on_disconnect
def inspect_disks(self, instance): def inspect_disks(self, instance):
domain = self._get_domain_not_shut_off_or_raise(instance) domain = self._get_domain_not_shut_off_or_raise(instance)
@ -167,34 +148,19 @@ class LibvirtInspector(virt_inspector.Inspector):
errors=block_stats[4]) errors=block_stats[4])
yield (disk, stats) yield (disk, stats)
@libvirt_utils.raise_nodata_if_unsupported("memory usge", False)
@libvirt_utils.retry_on_disconnect
def inspect_memory_usage(self, instance, duration=None): def inspect_memory_usage(self, instance, duration=None):
instance_name = util.instance_name(instance)
domain = self._get_domain_not_shut_off_or_raise(instance) domain = self._get_domain_not_shut_off_or_raise(instance)
try: memory_stats = domain.memoryStats()
memory_stats = domain.memoryStats() memory_used = (memory_stats['available'] -
if (memory_stats and memory_stats['unused'])
memory_stats.get('available') and # Stat provided from libvirt is in KB, converting it to MB.
memory_stats.get('unused')): memory_used = memory_used / units.Ki
memory_used = (memory_stats.get('available') - return virt_inspector.MemoryUsageStats(usage=memory_used)
memory_stats.get('unused'))
# Stat provided from libvirt is in KB, converting it to MB.
memory_used = memory_used / units.Ki
return virt_inspector.MemoryUsageStats(usage=memory_used)
else:
msg = _('Failed to inspect memory usage of instance '
'<name=%(name)s, id=%(id)s>, '
'can not get info from libvirt.') % {
'name': instance_name, 'id': instance.id}
raise virt_inspector.InstanceNoDataException(msg)
# memoryStats might launch an exception if the method is not supported
# by the underlying hypervisor being used by libvirt.
except libvirt.libvirtError as e:
msg = _('Failed to inspect memory usage of %(instance_uuid)s, '
'can not get info from libvirt: %(error)s') % {
'instance_uuid': instance.id, 'error': e}
raise virt_inspector.NoDataException(msg)
@libvirt_utils.retry_on_disconnect
def inspect_disk_info(self, instance): def inspect_disk_info(self, instance):
domain = self._get_domain_not_shut_off_or_raise(instance) domain = self._get_domain_not_shut_off_or_raise(instance)
tree = etree.fromstring(domain.XMLDesc(0)) tree = etree.fromstring(domain.XMLDesc(0))
@ -222,65 +188,32 @@ class LibvirtInspector(virt_inspector.Inspector):
physical=block_info[2]) physical=block_info[2])
yield (dsk, info) yield (dsk, info)
@libvirt_utils.retry_on_disconnect
def inspect_memory_resident(self, instance, duration=None): def inspect_memory_resident(self, instance, duration=None):
domain = self._get_domain_not_shut_off_or_raise(instance) domain = self._get_domain_not_shut_off_or_raise(instance)
memory = domain.memoryStats()['rss'] / units.Ki memory = domain.memoryStats()['rss'] / units.Ki
return virt_inspector.MemoryResidentStats(resident=memory) return virt_inspector.MemoryResidentStats(resident=memory)
@libvirt_utils.raise_nodata_if_unsupported("memory bandwidth")
@libvirt_utils.retry_on_disconnect
def inspect_memory_bandwidth(self, instance, duration=None): def inspect_memory_bandwidth(self, instance, duration=None):
domain = self._get_domain_not_shut_off_or_raise(instance) domain = self._get_domain_not_shut_off_or_raise(instance)
stats = self.connection.domainListGetStats(
[domain], libvirt.VIR_DOMAIN_STATS_PERF)
perf = stats[0][1]
return virt_inspector.MemoryBandwidthStats(total=perf["perf.mbmt"],
local=perf["perf.mbml"])
try: @libvirt_utils.raise_nodata_if_unsupported("perf events")
stats = self.connection.domainListGetStats( @libvirt_utils.retry_on_disconnect
[domain], libvirt.VIR_DOMAIN_STATS_PERF)
perf = stats[0][1]
return virt_inspector.MemoryBandwidthStats(total=perf["perf.mbmt"],
local=perf["perf.mbml"])
except (KeyError, AttributeError) as e:
# NOTE(sileht): KeyError if for libvirt >=2.0.0,<2.3.0, the perf
# subsystem ws existing but not these attributes
# https://github.com/libvirt/libvirt/commit/bae660869de0612bee2a740083fb494c27e3f80c
msg = _('Perf is not supported by current version of libvirt, and '
'failed to inspect memory bandwidth of %(instance_uuid)s, '
'can not get info from libvirt: %(error)s') % {
'instance_uuid': instance.id, 'error': e}
raise virt_inspector.NoDataException(msg)
# domainListGetStats might launch an exception if the method or
# mbmt/mbml perf event is not supported by the underlying hypervisor
# being used by libvirt.
except libvirt.libvirtError as e:
msg = _('Failed to inspect memory bandwidth of %(instance_uuid)s, '
'can not get info from libvirt: %(error)s') % {
'instance_uuid': instance.id, 'error': e}
raise virt_inspector.NoDataException(msg)
def inspect_perf_events(self, instance, duration=None): def inspect_perf_events(self, instance, duration=None):
domain = self._get_domain_not_shut_off_or_raise(instance) domain = self._get_domain_not_shut_off_or_raise(instance)
try: stats = self.connection.domainListGetStats(
stats = self.connection.domainListGetStats( [domain], libvirt.VIR_DOMAIN_STATS_PERF)
[domain], libvirt.VIR_DOMAIN_STATS_PERF) perf = stats[0][1]
perf = stats[0][1] return virt_inspector.PerfEventsStats(
return virt_inspector.PerfEventsStats( cpu_cycles=perf["perf.cpu_cycles"],
cpu_cycles=perf["perf.cpu_cycles"], instructions=perf["perf.instructions"],
instructions=perf["perf.instructions"], cache_references=perf["perf.cache_references"],
cache_references=perf["perf.cache_references"], cache_misses=perf["perf.cache_misses"])
cache_misses=perf["perf.cache_misses"])
# NOTE(sileht): KeyError if for libvirt >=2.0.0,<2.3.0, the perf
# subsystem ws existing but not these attributes
# https://github.com/libvirt/libvirt/commit/bae660869de0612bee2a740083fb494c27e3f80c
except (AttributeError, KeyError) as e:
msg = _LE('Perf is not supported by current version of libvirt, '
'and failed to inspect perf events of '
'%(instance_uuid)s, can not get info from libvirt: '
'%(error)s') % {
'instance_uuid': instance.id, 'error': e}
raise virt_inspector.NoDataException(msg)
# domainListGetStats might launch an exception if the method or
# mbmt/mbml perf event is not supported by the underlying hypervisor
# being used by libvirt.
except libvirt.libvirtError as e:
msg = _LE('Failed to inspect perf events of %(instance_uuid)s, '
'can not get info from libvirt: %(error)s') % {
'instance_uuid': instance.id, 'error': e}
raise virt_inspector.NoDataException(msg)

67
ceilometer/compute/virt/libvirt/utils.py
View File

@ -15,12 +15,16 @@
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log as logging from oslo_log import log as logging
import tenacity
try: try:
import libvirt import libvirt
except ImportError: except ImportError:
libvirt = None libvirt = None
from ceilometer.compute.virt import inspector as virt_inspector
from ceilometer.i18n import _LE
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
OPTS = [ OPTS = [
@ -75,7 +79,7 @@ LIBVIRT_STATUS = {
} }
def get_libvirt_connection(conf): def new_libvirt_connection(conf):
if not libvirt: if not libvirt:
raise ImportError("python-libvirt module is missing") raise ImportError("python-libvirt module is missing")
uri = (conf.libvirt_uri or LIBVIRT_PER_TYPE_URIS.get(conf.libvirt_type, uri = (conf.libvirt_uri or LIBVIRT_PER_TYPE_URIS.get(conf.libvirt_type,
@ -84,21 +88,46 @@ def get_libvirt_connection(conf):
return libvirt.openReadOnly(uri) return libvirt.openReadOnly(uri)
def retry_on_disconnect(function): def refresh_libvirt_connection(conf, klass):
def decorator(self, *args, **kwargs): connection = getattr(klass, '_libvirt_connection', None)
try: if not connection or not connection.isAlive():
return function(self, *args, **kwargs) connection = new_libvirt_connection(conf)
except ImportError: setattr(klass, '_libvirt_connection', connection)
# NOTE(sileht): in case of libvirt failed to be imported return connection
raise
except libvirt.libvirtError as e:
if (e.get_error_code() in (libvirt.VIR_ERR_SYSTEM_ERROR, def is_disconnection_exception(e):
libvirt.VIR_ERR_INTERNAL_ERROR) and return (isinstance(e, libvirt.libvirtError)
e.get_error_domain() in (libvirt.VIR_FROM_REMOTE, and e.get_error_code() in (libvirt.VIR_ERR_SYSTEM_ERROR,
libvirt.VIR_FROM_RPC)): libvirt.VIR_ERR_INTERNAL_ERROR)
LOG.debug('Connection to libvirt broken') and e.get_error_domain() in (libvirt.VIR_FROM_REMOTE,
self.connection = None libvirt.VIR_FROM_RPC))
return function(self, *args, **kwargs)
else:
raise retry_on_disconnect = tenacity.retry(
return decorator retry=tenacity.retry_if_exception(is_disconnection_exception),
stop=tenacity.stop_after_attempt(2))
class raise_nodata_if_unsupported(object):
def __init__(self, meter, permanent=True):
self.meter = meter
self.permanent = permanent
def __call__(self, method):
def inner(in_self, instance, *args, **kwargs):
try:
return method(in_self, instance, *args, **kwargs)
except (libvirt.libvirtError, KeyError, AttributeError) as e:
# NOTE(sileht): At this point libvirt connection error
# have been reraise as tenacity.RetryError()
msg = _LE('Failed to inspect %(meter)s of %(instance_uuid)s, '
'can not get info from libvirt: %(error)s') % {
"meter": self.meter,
"instance_uuid": instance.id,
"error": e}
if self.permanent:
raise virt_inspector.NoDataException(msg)
else:
raise virt_inspector.InstanceNoDataException(msg)
return inner

12
ceilometer/tests/unit/compute/pollsters/base.py
View File

@ -46,16 +46,14 @@ class TestPollsterBase(base.BaseTestCase):
'metering.stack': '2cadc4b4-8789-123c-b4eg-edd2f0a9c128', 'metering.stack': '2cadc4b4-8789-123c-b4eg-edd2f0a9c128',
'project_cos': 'dev'} 'project_cos': 'dev'}
patch_virt = mockpatch.Patch( self.useFixture(mockpatch.Patch(
'ceilometer.compute.virt.inspector.get_hypervisor_inspector', 'ceilometer.compute.virt.inspector.get_hypervisor_inspector',
new=mock.Mock(return_value=self.inspector)) new=mock.Mock(return_value=self.inspector)))
self.useFixture(patch_virt)
# as we're having lazy hypervisor inspector singleton object in the # as we're having lazy hypervisor inspector singleton object in the
# base compute pollster class, that leads to the fact that we # base compute pollster class, that leads to the fact that we
# need to mock all this class property to avoid context sharing between # need to mock all this class property to avoid context sharing between
# the tests # the tests
patch_inspector = mockpatch.Patch( self.useFixture(mockpatch.Patch(
'ceilometer.compute.pollsters.BaseComputePollster.inspector', 'ceilometer.compute.pollsters.BaseComputePollster._get_inspector',
self.inspector) return_value=self.inspector))
self.useFixture(patch_inspector)

376
ceilometer/tests/unit/compute/virt/libvirt/test_inspector.py
View File

@ -14,11 +14,6 @@
"""Tests for libvirt inspector. """Tests for libvirt inspector.
""" """
try:
import contextlib2 as contextlib # for Python < 3.3
except ImportError:
import contextlib
import fixtures import fixtures
import mock import mock
from oslo_config import fixture as fixture_config from oslo_config import fixture as fixture_config
@ -30,64 +25,66 @@ from ceilometer.compute.virt.libvirt import inspector as libvirt_inspector
from ceilometer.compute.virt.libvirt import utils from ceilometer.compute.virt.libvirt import utils
class TestLibvirtInspection(base.BaseTestCase): class FakeLibvirtError(Exception):
pass
class fakeLibvirtError(Exception):
pass class VMInstance(object):
id = 'ff58e738-12f4-4c58-acde-77617b68da56'
name = 'instance-00000001'
class TestLibvirtInspection(base.BaseTestCase):
def setUp(self): def setUp(self):
super(TestLibvirtInspection, self).setUp() super(TestLibvirtInspection, self).setUp()
self.CONF = self.useFixture(fixture_config.Config()).conf self.CONF = self.useFixture(fixture_config.Config()).conf
class VMInstance(object): self.instance = VMInstance()
id = 'ff58e738-12f4-4c58-acde-77617b68da56'
name = 'instance-00000001'
self.instance = VMInstance
self.inspector = libvirt_inspector.LibvirtInspector(self.CONF)
libvirt_inspector.libvirt = mock.Mock() libvirt_inspector.libvirt = mock.Mock()
libvirt_inspector.libvirt.VIR_DOMAIN_SHUTOFF = 5 libvirt_inspector.libvirt.VIR_DOMAIN_SHUTOFF = 5
libvirt_inspector.libvirt.libvirtError = self.fakeLibvirtError libvirt_inspector.libvirt.libvirtError = FakeLibvirtError
utils.libvirt = libvirt_inspector.libvirt utils.libvirt = libvirt_inspector.libvirt
with mock.patch('ceilometer.compute.virt.libvirt.utils.'
'refresh_libvirt_connection', return_value=None):
self.inspector = libvirt_inspector.LibvirtInspector(self.CONF)
self.domain = mock.Mock() self.domain = mock.Mock()
self.addCleanup(mock.patch.stopall)
def test_inspect_cpus(self): def test_inspect_cpus(self):
fake_stats = [({}, {'cpu.time': 999999, 'vcpu.current': 2})] domain = mock.Mock()
with contextlib.ExitStack() as stack: domain.info.return_value = (0, 0, 0, None, None)
stack.enter_context(mock.patch.object(self.inspector.connection, conn = mock.Mock()
'lookupByUUIDString', conn.lookupByUUIDString.return_value = domain
return_value=self.domain)) conn.domainListGetStats.return_value = [({}, {'cpu.time': 999999,
stack.enter_context(mock.patch.object(self.domain, 'info', 'vcpu.current': 2})]
return_value=(0, 0, 0,
None, None))) with mock.patch('ceilometer.compute.virt.libvirt.utils.'
stack.enter_context(mock.patch.object(self.inspector.connection, 'refresh_libvirt_connection', return_value=conn):
'domainListGetStats',
return_value=fake_stats))
cpu_info = self.inspector.inspect_cpus(self.instance) cpu_info = self.inspector.inspect_cpus(self.instance)
self.assertEqual(2, cpu_info.number) self.assertEqual(2, cpu_info.number)
self.assertEqual(999999, cpu_info.time) self.assertEqual(999999, cpu_info.time)
def test_inspect_cpus_with_domain_shutoff(self): def test_inspect_cpus_with_domain_shutoff(self):
connection = self.inspector.connection domain = mock.Mock()
with mock.patch.object(connection, 'lookupByUUIDString', domain.info.return_value = (5, 0, 0, 2, 999999)
return_value=self.domain): conn = mock.Mock()
with mock.patch.object(self.domain, 'info', conn.lookupByUUIDString.return_value = domain
return_value=(5, 0, 0,
2, 999999)): with mock.patch('ceilometer.compute.virt.libvirt.utils.'
self.assertRaises(virt_inspector.InstanceShutOffException, 'refresh_libvirt_connection', return_value=conn):
self.inspector.inspect_cpus, self.assertRaises(virt_inspector.InstanceShutOffException,
self.instance) self.inspector.inspect_cpus,
self.instance)
def test_inspect_cpu_l3_cache(self): def test_inspect_cpu_l3_cache(self):
fake_stats = [({}, {'perf.cmt': 90112})] domain = mock.Mock()
connection = self.inspector.connection domain.info.return_value = (0, 0, 0, 2, 999999)
with contextlib.ExitStack() as stack: conn = mock.Mock()
stack.enter_context(mock.patch.object(connection, conn.domainListGetStats.return_value = [({}, {'perf.cmt': 90112})]
'lookupByUUIDString', conn.lookupByUUIDString.return_value = domain
return_value=self.domain))
stack.enter_context(mock.patch.object(connection, with mock.patch('ceilometer.compute.virt.libvirt.utils.'
'domainListGetStats', 'refresh_libvirt_connection', return_value=conn):
return_value=fake_stats))
cpu_info = self.inspector.inspect_cpu_l3_cache(self.instance) cpu_info = self.inspector.inspect_cpu_l3_cache(self.instance)
self.assertEqual(90112, cpu_info.l3_cache_usage) self.assertEqual(90112, cpu_info.l3_cache_usage)
@ -156,19 +153,15 @@ class TestLibvirtInspection(base.BaseTestCase):
} }
interfaceStats = interface_stats.__getitem__ interfaceStats = interface_stats.__getitem__
connection = self.inspector.connection domain = mock.Mock()
with contextlib.ExitStack() as stack: domain.XMLDesc.return_value = dom_xml
stack.enter_context(mock.patch.object(connection, domain.info.return_value = (0, 0, 0, 2, 999999)
'lookupByUUIDString', domain.interfaceStats.side_effect = interfaceStats
return_value=self.domain)) conn = mock.Mock()
stack.enter_context(mock.patch.object(self.domain, 'XMLDesc', conn.lookupByUUIDString.return_value = domain
return_value=dom_xml))
stack.enter_context(mock.patch.object(self.domain, with mock.patch('ceilometer.compute.virt.libvirt.utils.'
'interfaceStats', 'refresh_libvirt_connection', return_value=conn):
side_effect=interfaceStats))
stack.enter_context(mock.patch.object(self.domain, 'info',
return_value=(0, 0, 0,
2, 999999)))
interfaces = list(self.inspector.inspect_vnics(self.instance)) interfaces = list(self.inspector.inspect_vnics(self.instance))
self.assertEqual(3, len(interfaces)) self.assertEqual(3, len(interfaces))
@ -209,14 +202,13 @@ class TestLibvirtInspection(base.BaseTestCase):
self.assertEqual(12, info2.tx_packets) self.assertEqual(12, info2.tx_packets)
def test_inspect_vnics_with_domain_shutoff(self): def test_inspect_vnics_with_domain_shutoff(self):
connection = self.inspector.connection domain = mock.Mock()
with contextlib.ExitStack() as stack: domain.info.return_value = (5, 0, 0, 2, 999999)
stack.enter_context(mock.patch.object(connection, conn = mock.Mock()
'lookupByUUIDString', conn.lookupByUUIDString.return_value = domain
return_value=self.domain))
stack.enter_context(mock.patch.object(self.domain, 'info', with mock.patch('ceilometer.compute.virt.libvirt.utils.'
return_value=(5, 0, 0, 'refresh_libvirt_connection', return_value=conn):
2, 999999)))
inspect = self.inspector.inspect_vnics inspect = self.inspector.inspect_vnics
self.assertRaises(virt_inspector.InstanceShutOffException, self.assertRaises(virt_inspector.InstanceShutOffException,
list, inspect(self.instance)) list, inspect(self.instance))
@ -236,19 +228,15 @@ class TestLibvirtInspection(base.BaseTestCase):
</devices> </devices>
</domain> </domain>
""" """
domain = mock.Mock()
domain.XMLDesc.return_value = dom_xml
domain.info.return_value = (0, 0, 0, 2, 999999)
domain.blockStats.return_value = (1, 2, 3, 4, -1)
conn = mock.Mock()
conn.lookupByUUIDString.return_value = domain
with contextlib.ExitStack() as stack: with mock.patch('ceilometer.compute.virt.libvirt.utils.'
stack.enter_context(mock.patch.object(self.inspector.connection, 'refresh_libvirt_connection', return_value=conn):
'lookupByUUIDString',
return_value=self.domain))
stack.enter_context(mock.patch.object(self.domain, 'XMLDesc',
return_value=dom_xml))
stack.enter_context(mock.patch.object(self.domain, 'blockStats',
return_value=(1, 2, 3,
4, -1)))
stack.enter_context(mock.patch.object(self.domain, 'info',
return_value=(0, 0, 0,
2, 999999)))
disks = list(self.inspector.inspect_disks(self.instance)) disks = list(self.inspector.inspect_disks(self.instance))
self.assertEqual(1, len(disks)) self.assertEqual(1, len(disks))
@ -260,31 +248,28 @@ class TestLibvirtInspection(base.BaseTestCase):
self.assertEqual(4, info0.write_bytes) self.assertEqual(4, info0.write_bytes)
def test_inspect_disks_with_domain_shutoff(self): def test_inspect_disks_with_domain_shutoff(self):
connection = self.inspector.connection domain = mock.Mock()
with contextlib.ExitStack() as stack: domain.info.return_value = (5, 0, 0, 2, 999999)
stack.enter_context(mock.patch.object(connection, conn = mock.Mock()
'lookupByUUIDString', conn.lookupByUUIDString.return_value = domain
return_value=self.domain))
stack.enter_context(mock.patch.object(self.domain, 'info', with mock.patch('ceilometer.compute.virt.libvirt.utils.'
return_value=(5, 0, 0, 'refresh_libvirt_connection', return_value=conn):
2, 999999)))
inspect = self.inspector.inspect_disks inspect = self.inspector.inspect_disks
self.assertRaises(virt_inspector.InstanceShutOffException, self.assertRaises(virt_inspector.InstanceShutOffException,
list, inspect(self.instance)) list, inspect(self.instance))
def test_inspect_memory_usage(self): def test_inspect_memory_usage(self):
fake_memory_stats = {'available': 51200, 'unused': 25600} domain = mock.Mock()
connection = self.inspector.connection domain.info.return_value = (0, 0, 0, 2, 999999)
with mock.patch.object(connection, 'lookupByUUIDString', domain.memoryStats.return_value = {'available': 51200, 'unused': 25600}
return_value=self.domain): conn = mock.Mock()
with mock.patch.object(self.domain, 'info', conn.lookupByUUIDString.return_value = domain
return_value=(0, 0, 51200,
2, 999999)): with mock.patch('ceilometer.compute.virt.libvirt.utils.'
with mock.patch.object(self.domain, 'memoryStats', 'refresh_libvirt_connection', return_value=conn):
return_value=fake_memory_stats): memory = self.inspector.inspect_memory_usage(self.instance)
memory = self.inspector.inspect_memory_usage( self.assertEqual(25600 / units.Ki, memory.usage)
self.instance)
self.assertEqual(25600 / units.Ki, memory.usage)
def test_inspect_disk_info(self): def test_inspect_disk_info(self):
dom_xml = """ dom_xml = """
@ -301,19 +286,15 @@ class TestLibvirtInspection(base.BaseTestCase):
</devices> </devices>
</domain> </domain>
""" """
domain = mock.Mock()
domain.XMLDesc.return_value = dom_xml
domain.blockInfo.return_value = (1, 2, 3, -1)
domain.info.return_value = (0, 0, 0, 2, 999999)
conn = mock.Mock()
conn.lookupByUUIDString.return_value = domain
with contextlib.ExitStack() as stack: with mock.patch('ceilometer.compute.virt.libvirt.utils.'
stack.enter_context(mock.patch.object(self.inspector.connection, 'refresh_libvirt_connection', return_value=conn):
'lookupByUUIDString',
return_value=self.domain))
stack.enter_context(mock.patch.object(self.domain, 'XMLDesc',
return_value=dom_xml))
stack.enter_context(mock.patch.object(self.domain, 'blockInfo',
return_value=(1, 2, 3,
-1)))
stack.enter_context(mock.patch.object(self.domain, 'info',
return_value=(0, 0, 0,
2, 999999)))
disks = list(self.inspector.inspect_disk_info(self.instance)) disks = list(self.inspector.inspect_disk_info(self.instance))
self.assertEqual(1, len(disks)) self.assertEqual(1, len(disks))
@ -338,21 +319,16 @@ class TestLibvirtInspection(base.BaseTestCase):
</devices> </devices>
</domain> </domain>
""" """
domain = mock.Mock()
domain.XMLDesc.return_value = dom_xml
domain.blockInfo.return_value = (1, 2, 3, -1)
domain.info.return_value = (0, 0, 0, 2, 999999)
conn = mock.Mock()
conn.lookupByUUIDString.return_value = domain
with contextlib.ExitStack() as stack: with mock.patch('ceilometer.compute.virt.libvirt.utils.'
stack.enter_context(mock.patch.object(self.inspector.connection, 'refresh_libvirt_connection', return_value=conn):
'lookupByUUIDString',
return_value=self.domain))
stack.enter_context(mock.patch.object(self.domain, 'XMLDesc',
return_value=dom_xml))
stack.enter_context(mock.patch.object(self.domain, 'blockInfo',
return_value=(1, 2, 3,
-1)))
stack.enter_context(mock.patch.object(self.domain, 'info',
return_value=(0, 0, 0,
2, 999999)))
disks = list(self.inspector.inspect_disk_info(self.instance)) disks = list(self.inspector.inspect_disk_info(self.instance))
self.assertEqual(0, len(disks)) self.assertEqual(0, len(disks))
def test_inspect_disk_info_without_source_element(self): def test_inspect_disk_info_without_source_element(self):
@ -371,111 +347,103 @@ class TestLibvirtInspection(base.BaseTestCase):
</devices> </devices>
</domain> </domain>
""" """
domain = mock.Mock()
domain.XMLDesc.return_value = dom_xml
domain.blockInfo.return_value = (1, 2, 3, -1)
domain.info.return_value = (0, 0, 0, 2, 999999)
conn = mock.Mock()
conn.lookupByUUIDString.return_value = domain
with contextlib.ExitStack() as stack: with mock.patch('ceilometer.compute.virt.libvirt.utils.'
stack.enter_context(mock.patch.object(self.inspector.connection, 'refresh_libvirt_connection', return_value=conn):
'lookupByUUIDString',
return_value=self.domain))
stack.enter_context(mock.patch.object(self.domain, 'XMLDesc',
return_value=dom_xml))
stack.enter_context(mock.patch.object(self.domain, 'blockInfo',
return_value=(1, 2, 3,
-1)))
stack.enter_context(mock.patch.object(self.domain, 'info',
return_value=(0, 0, 0,
2, 999999)))
disks = list(self.inspector.inspect_disk_info(self.instance)) disks = list(self.inspector.inspect_disk_info(self.instance))
self.assertEqual(0, len(disks)) self.assertEqual(0, len(disks))
def test_inspect_memory_usage_with_domain_shutoff(self): def test_inspect_memory_usage_with_domain_shutoff(self):
connection = self.inspector.connection domain = mock.Mock()
with mock.patch.object(connection, 'lookupByUUIDString', domain.info.return_value = (5, 0, 51200, 2, 999999)
return_value=self.domain): conn = mock.Mock()
with mock.patch.object(self.domain, 'info', conn.lookupByUUIDString.return_value = domain
return_value=(5, 0, 0,
2, 999999)): with mock.patch('ceilometer.compute.virt.libvirt.utils.'
self.assertRaises(virt_inspector.InstanceShutOffException, 'refresh_libvirt_connection', return_value=conn):
self.inspector.inspect_memory_usage, self.assertRaises(virt_inspector.InstanceShutOffException,
self.instance) self.inspector.inspect_memory_usage,
self.instance)
def test_inspect_memory_usage_with_empty_stats(self): def test_inspect_memory_usage_with_empty_stats(self):
connection = self.inspector.connection domain = mock.Mock()
with mock.patch.object(connection, 'lookupByUUIDString', domain.info.return_value = (0, 0, 51200, 2, 999999)
return_value=self.domain): domain.memoryStats.return_value = {}
with mock.patch.object(self.domain, 'info', conn = mock.Mock()
return_value=(0, 0, 51200, conn.lookupByUUIDString.return_value = domain
2, 999999)):
with mock.patch.object(self.domain, 'memoryStats', with mock.patch('ceilometer.compute.virt.libvirt.utils.'
return_value={}): 'refresh_libvirt_connection', return_value=conn):
self.assertRaises(virt_inspector.InstanceNoDataException, self.assertRaises(virt_inspector.InstanceNoDataException,
self.inspector.inspect_memory_usage, self.inspector.inspect_memory_usage,
self.instance) self.instance)
def test_inspect_memory_bandwidth(self): def test_inspect_memory_bandwidth(self):
fake_stats = [({}, {'perf.mbmt': 1892352, 'perf.mbml': 1802240})] domain = mock.Mock()
connection = self.inspector.connection domain.info.return_value = (0, 0, 51200, 2, 999999)
with mock.patch.object(connection, 'lookupByUUIDString', conn = mock.Mock()
return_value=self.domain): conn.domainListGetStats.return_value = [
with mock.patch.object(self.domain, 'info', ({}, {'perf.mbmt': 1892352, 'perf.mbml': 1802240})]
return_value=(0, 0, 51200, conn.lookupByUUIDString.return_value = domain
2, 999999)):
with mock.patch.object(connection, 'domainListGetStats', with mock.patch('ceilometer.compute.virt.libvirt.utils.'
return_value=fake_stats): 'refresh_libvirt_connection', return_value=conn):
mb = self.inspector.inspect_memory_bandwidth(self.instance) mb = self.inspector.inspect_memory_bandwidth(self.instance)
self.assertEqual(1892352, mb.total) self.assertEqual(1892352, mb.total)
self.assertEqual(1802240, mb.local) self.assertEqual(1802240, mb.local)
def test_inspect_perf_events(self): def test_inspect_perf_events(self):
fake_stats = [({}, {'perf.cpu_cycles': 7259361, domain = mock.Mock()
'perf.instructions': 8815623, domain.info.return_value = (0, 0, 51200, 2, 999999)
'perf.cache_references': 74184, conn = mock.Mock()
'perf.cache_misses': 16737})] conn.domainListGetStats.return_value = [
connection = self.inspector.connection ({}, {'perf.cpu_cycles': 7259361,
with mock.patch.object(connection, 'lookupByUUIDString', 'perf.instructions': 8815623,
return_value=self.domain): 'perf.cache_references': 74184,
with mock.patch.object(self.domain, 'info', 'perf.cache_misses': 16737})]
return_value=(0, 0, 51200, conn.lookupByUUIDString.return_value = domain
2, 999999)):
with mock.patch.object(connection, 'domainListGetStats', with mock.patch('ceilometer.compute.virt.libvirt.utils.'
return_value=fake_stats): 'refresh_libvirt_connection', return_value=conn):
pe = self.inspector.inspect_perf_events(self.instance) pe = self.inspector.inspect_perf_events(self.instance)
self.assertEqual(7259361, pe.cpu_cycles) self.assertEqual(7259361, pe.cpu_cycles)
self.assertEqual(8815623, pe.instructions) self.assertEqual(8815623, pe.instructions)
self.assertEqual(74184, pe.cache_references) self.assertEqual(74184, pe.cache_references)
self.assertEqual(16737, pe.cache_misses) self.assertEqual(16737, pe.cache_misses)
def test_inspect_perf_events_libvirt_less_than_2_3_0(self): def test_inspect_perf_events_libvirt_less_than_2_3_0(self):
fake_stats = [({}, {})] domain = mock.Mock()
connection = self.inspector.connection domain.info.return_value = (0, 0, 51200, 2, 999999)
with mock.patch.object(connection, 'lookupByUUIDString', conn = mock.Mock()
return_value=self.domain): conn.domainListGetStats.return_value = [({}, {})]
with mock.patch.object(self.domain, 'info', conn.lookupByUUIDString.return_value = domain
return_value=(0, 0, 51200,
2, 999999)): with mock.patch('ceilometer.compute.virt.libvirt.utils.'
with mock.patch.object(connection, 'domainListGetStats', 'refresh_libvirt_connection', return_value=conn):
return_value=fake_stats): self.assertRaises(virt_inspector.NoDataException,
self.assertRaises(virt_inspector.NoDataException, self.inspector.inspect_perf_events,
self.inspector.inspect_perf_events, self.instance)
self.instance)
class TestLibvirtInspectionWithError(base.BaseTestCase): class TestLibvirtInspectionWithError(base.BaseTestCase):
class fakeLibvirtError(Exception):
pass
def setUp(self): def setUp(self):
super(TestLibvirtInspectionWithError, self).setUp() super(TestLibvirtInspectionWithError, self).setUp()
self.CONF = self.useFixture(fixture_config.Config()).conf self.CONF = self.useFixture(fixture_config.Config()).conf
self.inspector = libvirt_inspector.LibvirtInspector(self.CONF)
self.useFixture(fixtures.MonkeyPatch( self.useFixture(fixtures.MonkeyPatch(
'ceilometer.compute.virt.libvirt.inspector.' 'ceilometer.compute.virt.libvirt.utils.'
'LibvirtInspector.connection', 'refresh_libvirt_connection',
mock.MagicMock(side_effect=Exception('dummy')))) mock.MagicMock(side_effect=[None, Exception('dummy')])))
libvirt_inspector.libvirt = mock.Mock() libvirt_inspector.libvirt = mock.Mock()
libvirt_inspector.libvirt.libvirtError = self.fakeLibvirtError libvirt_inspector.libvirt.libvirtError = FakeLibvirtError
utils.libvirt = libvirt_inspector.libvirt utils.libvirt = libvirt_inspector.libvirt
self.inspector = libvirt_inspector.LibvirtInspector(self.CONF)
def test_inspect_unknown_error(self): def test_inspect_unknown_error(self):
self.assertRaises(virt_inspector.InspectorException, self.assertRaises(virt_inspector.InspectorException,

1
requirements.txt
View File

@ -43,6 +43,7 @@ six>=1.9.0 # MIT
SQLAlchemy<1.1.0,>=1.0.10 # MIT SQLAlchemy<1.1.0,>=1.0.10 # MIT
sqlalchemy-migrate>=0.9.6 # Apache-2.0 sqlalchemy-migrate>=0.9.6 # Apache-2.0
stevedore>=1.9.0 # Apache-2.0 stevedore>=1.9.0 # Apache-2.0
tenacity>=3.2.1 # Apache-2.0
tooz>=1.47.0 # Apache-2.0 tooz>=1.47.0 # Apache-2.0
WebOb>=1.5.0 # MIT WebOb>=1.5.0 # MIT
WSME>=0.8 # MIT WSME>=0.8 # MIT

1
test-requirements.txt
View File

@ -2,7 +2,6 @@
# of appearance. Changing the order has an impact on the overall integration # of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later. # process, which may cause wedges in the gate later.
contextlib2>=0.4.0 # PSF License
coverage>=3.6 # Apache-2.0 coverage>=3.6 # Apache-2.0
fixtures<2.0,>=1.3.1 # Apache-2.0/BSD fixtures<2.0,>=1.3.1 # Apache-2.0/BSD
happybase!=0.7,>=0.5,<1.0.0;python_version=='2.7' # MIT happybase!=0.7,>=0.5,<1.0.0;python_version=='2.7' # MIT