Use stevedore for loading monitor extensions

The compute monitor plugins were being loaded using the nova.loadables
module. This patch uses the stevedore library to load monitor
extensions. We keep the same semantics as the previous loading
mechanism: we use the CONF.compute_monitors configuration option to
winnow the set of all monitor plugins, and we ensure that no two
monitors that return the same set of metrics will be loaded.

However, this patch deprecates the CONF.compute_available_monitors
configuration option, since stevedore and setuptools entry points now
allow a set of plugins to be specified without any further configuration
options.

Change-Id: I97bf8bcd43faf9f3fe40983497c2360233d5f599
Fixes-bug: 1468012
DocImpact: deprecates the CONF.compute_available_monitors option.

nova/compute/monitors/__init__.py

@@ -19,20 +19,24 @@ Resource monitor API specification.
 
 from oslo_config import cfg
 from oslo_log import log as logging
+from stevedore import enabled
 
-import nova.compute.monitors.base
 from nova.i18n import _LW
-from nova import loadables
 
 compute_monitors_opts = [
     cfg.MultiStrOpt('compute_available_monitors',
-                    default=['nova.compute.monitors.all_monitors'],
+                    deprecated_for_removal=True,
+                    default=None,
                     help='Monitor classes available to the compute which may '
-                         'be specified more than once.'),
+                         'be specified more than once. This option is '
+                         'DEPRECATED and no longer used. Use setuptools entry '
+                         'points to list available monitor plugins.'),
     cfg.ListOpt('compute_monitors',
                 default=[],
                 help='A list of monitors that can be used for getting '
-                     'compute metrics.'),
+                     'compute metrics. You can use the alias/name from '
+                     'the setuptools entry points for nova.compute.monitors.* '
+                     'namespaces.'),
     ]
 
 CONF = cfg.CONF
@@ -40,64 +44,42 @@ CONF.register_opts(compute_monitors_opts)
 LOG = logging.getLogger(__name__)
 
 
-# TODO(jaypipes): Replace the use of loadables with stevedore.
+class MonitorHandler(object):
-class ResourceMonitorHandler(loadables.BaseLoader):
-    """Base class to handle loading monitor classes.
-    """
-    def __init__(self):
-        super(ResourceMonitorHandler, self).__init__(
-                nova.compute.monitors.base.MonitorBase)
 
-    def choose_monitors(self, manager):
+    def __init__(self, resource_tracker):
-        """This function checks the monitor names and metrics names against a
+        self.cpu_monitor_loaded = False
-        predefined set of acceptable monitors.
-        """
-        monitor_classes = self.get_matching_classes(
-             CONF.compute_available_monitors)
-        monitor_class_map = {cls.__name__: cls for cls in monitor_classes}
-        monitor_cls_names = CONF.compute_monitors
-        good_monitors = []
-        bad_monitors = []
-        metric_names = set()
-        for monitor_name in monitor_cls_names:
-            if monitor_name not in monitor_class_map:
-                bad_monitors.append(monitor_name)
-                continue
 
-            try:
+        ns = 'nova.compute.monitors.cpu'
-                # make sure different monitors do not have the same
+        cpu_plugin_mgr = enabled.EnabledExtensionManager(
-                # metric name
+                namespace=ns,
-                monitor = monitor_class_map[monitor_name](manager)
+                invoke_on_load=True,
-                metric_names_tmp = monitor.get_metric_names()
+                check_func=self.check_enabled_cpu_monitor,
-                overlap = metric_names & metric_names_tmp
+                invoke_args=(resource_tracker,)
-                if not overlap:
+        )
-                    metric_names = metric_names | metric_names_tmp
+        self.monitors = [obj.obj for obj in cpu_plugin_mgr]
-                    good_monitors.append(monitor)
+
-                else:
+    def check_enabled_cpu_monitor(self, ext):
-                    msg = (_LW("Excluding monitor %(monitor_name)s due to "
+        if self.cpu_monitor_loaded is not False:
-                               "metric name overlap; overlapping "
+            msg = _LW("Excluding CPU monitor %(monitor_name)s. Already "
-                               "metrics: %(overlap)s") %
+                      "loaded %(loaded_cpu_monitor)s.")
-                               {'monitor_name': monitor_name,
+            msg = msg % {
-                                'overlap': ', '.join(overlap)})
+                'monitor_name': ext.name,
+                'loaded_cpu_monitor': self.cpu_monitor_loaded
+            }
             LOG.warn(msg)
-                    bad_monitors.append(monitor_name)
+            return False
-            except Exception as ex:
+        # TODO(jaypipes): Right now, we only have CPU monitors, so we don't
-                msg = (_LW("Monitor %(monitor_name)s cannot be used: %(ex)s") %
+        # need to check if the plugin is a CPU monitor or not. Once non-CPU
-                          {'monitor_name': monitor_name, 'ex': ex})
+        # monitors are added, change this to check either the base class or
+        # the set of metric names returned to ensure only a single CPU
+        # monitor is loaded at any one time.
+        if ext.name in CONF.compute_monitors:
+            self.cpu_monitor_loaded = ext.name
+            return True
+        msg = _LW("Excluding CPU monitor %(monitor_name)s. Not in the "
+                  "list of enabled monitors (CONF.compute_monitors).")
+        msg = msg % {
+            'monitor_name': ext.name,
+        }
         LOG.warn(msg)
-                bad_monitors.append(monitor_name)
+        return False
-
-        if bad_monitors:
-            LOG.warning(_LW("The following monitors have been disabled: %s"),
-                        ', '.join(bad_monitors))
-
-        return good_monitors
-
-
-def all_monitors():
-    """Return a list of monitor classes found in this directory.
-
-    This method is used as the default for available monitors
-    and should return a list of all monitor classes available.
-    """
-    return ResourceMonitorHandler().get_all_classes()

nova/compute/monitors/cpu/virt_driver.py

@@ -33,10 +33,10 @@ LOG = logging.getLogger(__name__)
 class Monitor(base.CPUMonitorBase):
     """CPU monitor that uses the virt driver's get_host_cpu_stats() call."""
 
-    def __init__(self, compute_manager):
+    def __init__(self, resource_tracker):
-        super(Monitor, self).__init__(compute_manager)
+        super(Monitor, self).__init__(resource_tracker)
         self.source = CONF.compute_driver
-        self.driver = self.compute_manager.driver
+        self.driver = resource_tracker.driver
         self._data = {}
         self._cpu_stats = {}
 

nova/compute/resource_tracker.py

@@ -80,8 +80,8 @@ class ResourceTracker(object):
         self.tracked_instances = {}
         self.tracked_migrations = {}
         self.conductor_api = conductor.API()
-        monitor_handler = monitors.ResourceMonitorHandler()
+        monitor_handler = monitors.MonitorHandler(self)
-        self.monitors = monitor_handler.choose_monitors(self)
+        self.monitors = monitor_handler.monitors
         self.ext_resources_handler = \
             ext_resources.ResourceHandler(CONF.compute_resources)
         self.old_resources = objects.ComputeNode()

nova/tests/unit/compute/monitors/cpu/test_virt_driver.py

@@ -20,11 +20,7 @@ from nova import objects
 from nova import test
 
 
-class ComputeDriverCPUMonitorTestCase(test.NoDBTestCase):
+class FakeDriver(object):
-    def setUp(self):
-        super(ComputeDriverCPUMonitorTestCase, self).setUp()
-
-        class FakeDriver(object):
     def get_host_cpu_stats(self):
         return {'kernel': 5664160000000,
                 'idle': 1592705190000000,
@@ -32,13 +28,16 @@ class ComputeDriverCPUMonitorTestCase(test.NoDBTestCase):
                 'user': 26728850000000,
                 'iowait': 6121490000000}
 
-        class FakeComputeManager(object):
+
+class FakeResourceTracker(object):
     driver = FakeDriver()
 
-        self.monitor = virt_driver.Monitor(FakeComputeManager())
+
+class VirtDriverCPUMonitorTestCase(test.NoDBTestCase):
 
     def test_get_metric_names(self):
-        names = self.monitor.get_metric_names()
+        monitor = virt_driver.Monitor(FakeResourceTracker())
+        names = monitor.get_metric_names()
         self.assertEqual(10, len(names))
         self.assertIn("cpu.frequency", names)
         self.assertIn("cpu.user.time", names)
@@ -53,8 +52,9 @@ class ComputeDriverCPUMonitorTestCase(test.NoDBTestCase):
 
     def test_get_metrics(self):
         metrics = objects.MonitorMetricList()
-        self.monitor.add_metrics_to_list(metrics)
+        monitor = virt_driver.Monitor(FakeResourceTracker())
-        names = self.monitor.get_metric_names()
+        monitor.add_metrics_to_list(metrics)
+        names = monitor.get_metric_names()
         for metric in metrics.objects:
             self.assertIn(metric.name, names)
 

nova/tests/unit/compute/monitors/test_monitors.py

@@ -15,66 +15,25 @@
 
 """Tests for resource monitors."""
 
-from oslo_utils import timeutils
+import mock
 
 from nova.compute import monitors
-from nova.compute.monitors import base
-from nova.objects import fields
 from nova import test
 
 
-class CPUMonitor1(base.MonitorBase):
+class MonitorsTestCase(test.NoDBTestCase):
-
-    NOW_TS = timeutils.utcnow()
-
-    def __init__(self, *args):
-        super(CPUMonitor1, self).__init__(*args)
-        self.source = 'CPUMonitor1'
-
-    def get_metric_names(self):
-        return set([
-            fields.MonitorMetricType.CPU_FREQUENCY
-        ])
-
-    def get_metric(self, name):
-        return 100, CPUMonitor1.NOW_TS
-
-
-class CPUMonitor2(base.MonitorBase):
-
-    def get_metric_names(self):
-        return set([
-            fields.MonitorMetricType.CPU_FREQUENCY
-        ])
-
-    def get_metric(self, name):
-        # This should never be called since the CPU metrics overlap
-        # with the ones in the CPUMonitor1.
-        pass
-
-
-class ResourceMonitorsTestCase(test.NoDBTestCase):
     """Test case for monitors."""
 
-    def setUp(self):
+    @mock.patch('stevedore.enabled.EnabledExtensionManager')
-        super(ResourceMonitorsTestCase, self).setUp()
+    def test_check_enabled_cpu_monitor(self, _mock_ext_manager):
-        self.monitor_handler = monitors.ResourceMonitorHandler()
+        class FakeExt(object):
-        fake_monitors = [
+            def __init__(self, name):
-            'nova.tests.unit.compute.monitors.test_monitors.CPUMonitor1',
+                self.name = name
-            'nova.tests.unit.compute.monitors.test_monitors.CPUMonitor2']
-        self.flags(compute_available_monitors=fake_monitors)
 
-    def test_choose_monitors_not_found(self):
+        # We check to ensure only one CPU monitor is loaded...
-        self.flags(compute_monitors=['CPUMonitor1', 'CPUMonitorb'])
+        self.flags(compute_monitors=['cpu_mon1', 'cpu_mon2'])
-        monitor_classes = self.monitor_handler.choose_monitors(self)
+        handler = monitors.MonitorHandler(None)
-        self.assertEqual(len(monitor_classes), 1)
+        ext_cpu_mon1 = FakeExt('cpu_mon1')
-
+        ext_cpu_mon2 = FakeExt('cpu_mon2')
-    def test_choose_monitors_bad(self):
+        self.assertTrue(handler.check_enabled_cpu_monitor(ext_cpu_mon1))
-        self.flags(compute_monitors=['CPUMonitor1', 'CPUMonitor2'])
+        self.assertFalse(handler.check_enabled_cpu_monitor(ext_cpu_mon2))
-        monitor_classes = self.monitor_handler.choose_monitors(self)
-        self.assertEqual(len(monitor_classes), 1)
-
-    def test_choose_monitors_none(self):
-        self.flags(compute_monitors=[])
-        monitor_classes = self.monitor_handler.choose_monitors(self)
-        self.assertEqual(len(monitor_classes), 0)

nova/tests/unit/compute/test_resource_tracker.py

@@ -24,6 +24,7 @@ from oslo_config import cfg
 from oslo_serialization import jsonutils
 from oslo_utils import timeutils
 
+from nova.compute.monitors import base as monitor_base
 from nova.compute import resource_tracker
 from nova.compute import resources
 from nova.compute import task_states
@@ -36,7 +37,6 @@ from nova.objects import base as obj_base
 from nova.objects import pci_device_pool
 from nova import rpc
 from nova import test
-from nova.tests.unit.compute.monitors import test_monitors
 from nova.tests.unit.pci import fakes as pci_fakes
 from nova.virt import driver
 
@@ -201,7 +201,8 @@ class FakeVirtDriver(driver.ComputeDriver):
 
 class BaseTestCase(test.TestCase):
 
-    def setUp(self):
+    @mock.patch('stevedore.enabled.EnabledExtensionManager')
+    def setUp(self, _mock_ext_mgr):
         super(BaseTestCase, self).setUp()
 
         self.flags(reserved_host_disk_mb=0,
@@ -1276,10 +1277,6 @@ class OrphanTestCase(BaseTrackerTestCase):
 class ComputeMonitorTestCase(BaseTestCase):
     def setUp(self):
         super(ComputeMonitorTestCase, self).setUp()
-        fake_monitors = [
-            'nova.tests.unit.compute.monitors.test_monitors.CPUMonitor1',
-            'nova.tests.unit.compute.monitors.test_monitors.CPUMonitor2']
-        self.flags(compute_available_monitors=fake_monitors)
         self.tracker = self._tracker()
         self.node_name = 'nodename'
         self.user_id = 'fake'
@@ -1289,7 +1286,6 @@ class ComputeMonitorTestCase(BaseTestCase):
                                               self.project_id)
 
     def test_get_host_metrics_none(self):
-        self.flags(compute_monitors=[])
         self.tracker.monitors = []
         metrics = self.tracker._get_host_metrics(self.context,
                                                  self.node_name)
@@ -1307,9 +1303,21 @@ class ComputeMonitorTestCase(BaseTestCase):
         self.assertEqual(0, len(metrics))
 
     def test_get_host_metrics(self):
-        class1 = test_monitors.CPUMonitor1(self.tracker)
+        class FakeCPUMonitor(monitor_base.MonitorBase):
-        self.tracker.monitors = [class1]
 
+            NOW_TS = timeutils.utcnow()
+
+            def __init__(self, *args):
+                super(FakeCPUMonitor, self).__init__(*args)
+                self.source = 'FakeCPUMonitor'
+
+            def get_metric_names(self):
+                return set(["cpu.frequency"])
+
+            def get_metric(self, name):
+                return 100, self.NOW_TS
+
+        self.tracker.monitors = [FakeCPUMonitor(None)]
         mock_notifier = mock.Mock()
 
         with mock.patch.object(rpc, 'get_notifier',
@@ -1322,10 +1330,10 @@ class ComputeMonitorTestCase(BaseTestCase):
         expected_metrics = [
             {
                 'timestamp': timeutils.strtime(
-                    test_monitors.CPUMonitor1.NOW_TS),
+                    FakeCPUMonitor.NOW_TS),
                 'name': 'cpu.frequency',
                 'value': 100,
-                'source': 'CPUMonitor1'
+                'source': 'FakeCPUMonitor'
             },
         ]
 

setup.cfg

@@ -41,6 +41,8 @@ oslo.config.opts =
     nova.openstack.common.sslutils = nova.openstack.common.sslutils:list_opts
     nova.openstack.common.versionutils = nova.openstack.common.versionutils:list_opts
 
+nova.compute.monitors.cpu =
+    virt_driver = nova.compute.monitors.cpu.virt_driver:Monitor
 nova.compute.resources =
     vcpu = nova.compute.resources.vcpu:VCPU
 nova.image.download.modules =