Support to load pollsters extensions at runtime

Make AgentManager to load pollster extensions at runtime. So that a
pollster builder may build multiple pollster extensions to be used by
the AgentManager.

In this way, we don't need to change the existing pollsters interface in
order to support declarative pollster, which means a single pollster
class might gives more than one meters.

All pollsters which implements classmethod interface build_pollsters()
should be registered in the namesapce of ceilometer.builder.poll.xxx, so
that AgentManger can correctly find it and use to dynamically generate
multiple pollsters through the 'builder' logic.

Change-Id: If7f8f80301474e2684ea0f1bbdc69cc202b9ef7e
Implements: blueprint declarative-snmp-metrics

ceilometer/agent/base.py

@@ -227,11 +227,17 @@ class AgentManager(service_base.BaseService):
         # be passed
         extensions = (self._extensions('poll', namespace).extensions
                       for namespace in namespaces)
+        # get the extensions from pollster builder
+        extensions_fb = (self._extensions_from_builder('poll', namespace)
+                         for namespace in namespaces)
         if pollster_list:
             extensions = (moves.filter(_match, exts)
                           for exts in extensions)
+            extensions_fb = (moves.filter(_match, exts)
+                             for exts in extensions_fb)
 
-        self.extensions = list(itertools.chain(*list(extensions)))
+        self.extensions = list(itertools.chain(*list(extensions))) + list(
+            itertools.chain(*list(extensions_fb)))
 
         self.discovery_manager = self._extensions('discover')
         self.context = context.RequestContext('admin', 'admin', is_admin=True)
@@ -249,10 +255,7 @@ class AgentManager(service_base.BaseService):
             publisher_id="ceilometer.api")
 
     @staticmethod
-    def _extensions(category, agent_ns=None):
-        namespace = ('ceilometer.%s.%s' % (category, agent_ns) if agent_ns
-                     else 'ceilometer.%s' % category)
-
+    def _get_ext_mgr(namespace):
         def _catch_extension_load_error(mgr, ep, exc):
             # Extension raising ExtensionLoadError can be ignored,
             # and ignore anything we can't import as a safety measure.
@@ -260,10 +263,9 @@ class AgentManager(service_base.BaseService):
                 LOG.error(_("Skip loading extension for %s") % ep.name)
                 return
             if isinstance(exc, ImportError):
-                LOG.error(
-                    _("Failed to import extension for %(name)s: %(error)s"),
-                    {'name': ep.name, 'error': exc},
-                )
+                LOG.error(_("Failed to import extension for %(name)s: "
+                            "%(error)s"),
+                          {'name': ep.name, 'error': exc})
                 return
             raise exc
 
@@ -273,6 +275,26 @@ class AgentManager(service_base.BaseService):
             on_load_failure_callback=_catch_extension_load_error,
         )
 
+    def _extensions(self, category, agent_ns=None):
+        namespace = ('ceilometer.%s.%s' % (category, agent_ns) if agent_ns
+                     else 'ceilometer.%s' % category)
+        return self._get_ext_mgr(namespace)
+
+    def _extensions_from_builder(self, category, agent_ns=None):
+        ns = ('ceilometer.builder.%s.%s' % (category, agent_ns) if agent_ns
+              else 'ceilometer.builder.%s' % category)
+        mgr = self._get_ext_mgr(ns)
+
+        def _build(ext):
+            return ext.plugin.get_pollsters_extensions()
+
+        # NOTE: this seems a stevedore bug. if no extensions are found,
+        # map will raise runtimeError which is not documented.
+        if mgr.names():
+            return list(itertools.chain(*mgr.map(_build)))
+        else:
+            return []
+
     def join_partitioning_groups(self):
         self.groups = set([self.construct_group_id(d.obj.group_id)
                           for d in self.discovery_manager])

ceilometer/tests/unit/agent/test_manager.py

@@ -39,6 +39,12 @@ class PollingException(Exception):
     pass
 
 
+class TestPollsterBuilder(agentbase.TestPollster):
+    @classmethod
+    def build_pollsters(cls):
+        return [('builder1', cls()), ('builder2', cls())]
+
+
 @mock.patch('ceilometer.compute.pollsters.'
             'BaseComputePollster.setup_environment',
             mock.Mock(return_value=None))
@@ -126,6 +132,36 @@ class TestManager(base.BaseTestCase):
                           pollster_list=['disk.*'])
         manager.cfg.CONF.reset()
 
+    def test_builder(self):
+        @staticmethod
+        def fake_get_ext_mgr(namespace):
+            if 'builder' in namespace:
+                return extension.ExtensionManager.make_test_instance(
+                    [
+                        extension.Extension('builder',
+                                            None,
+                                            TestPollsterBuilder,
+                                            None),
+                    ]
+                )
+            else:
+                return extension.ExtensionManager.make_test_instance(
+                    [
+                        extension.Extension('test',
+                                            None,
+                                            None,
+                                            agentbase.TestPollster()),
+                    ]
+                )
+
+        with mock.patch.object(manager.AgentManager, '_get_ext_mgr',
+                               new=fake_get_ext_mgr):
+            mgr = manager.AgentManager(namespaces=['central'])
+            self.assertEqual(3, len(mgr.extensions))
+            for ext in mgr.extensions:
+                self.assertIn(ext.name, ['builder1', 'builder2', 'test'])
+                self.assertIsInstance(ext.obj, agentbase.TestPollster)
+
 
 class TestPollsterKeystone(agentbase.TestPollster):
     @plugin_base.check_keystone