Make reuse of builders/publishers inside other components easier.

Some Jenkins plugins depend on other plugins, and their configuration
section is a mix of both plugins.

For Jenkins Job Builder, that means reusing one component directly from
another one.

Driving the generation of XML markup is the job of Base._dispatch.
Unfortunately, components do not have access to their module object,
and even if their could, _dispatch would still be a non-public method.

Refactor Base._dispatch into ModuleRegistry.dispatch, which can be used
from any place where the parser is available.

Base and ModuleRegistry are extended so that the registry can discover
which entry point must be used for each module, if appropriate.
ModuleRegistry.dispatch signature can be simplified by dropping
component_list_type parameter.

Change-Id: Ie9d090817d0c2d464745b5634a22d3cea6a47ab1
Reviewed-on: https://review.openstack.org/26051
Reviewed-by: James E. Blair <corvus@inaugust.com>
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins

doc/source/extending.rst

@@ -65,3 +65,15 @@ wanted to add a new builder, all you need to do is write a function
 that conforms to the :ref:`Component Interface <component_interface>`,
 and then add that function to the appropriate entry point (via a
 setup.py file).
+
+.. _module_registry:
+
+Module Registry
+---------------
+
+All modules and their associated components are registered in the
+module registry. It can be accessed either from modules via the registry
+field, or via the parser parameter of components.
+
+.. autoclass:: jenkins_jobs.builder.ModuleRegistry
+   :members:

jenkins_jobs/builder.py

@@ -209,6 +209,7 @@ class YamlParser(object):
 class ModuleRegistry(object):
     def __init__(self, config):
         self.modules = []
+        self.modules_by_component_type = {}
         self.handlers = {}
         self.global_config = config
 
@@ -218,6 +219,8 @@ class ModuleRegistry(object):
             mod = Mod(self)
             self.modules.append(mod)
             self.modules.sort(lambda a, b: cmp(a.sequence, b.sequence))
+            if mod.component_type is not None:
+                self.modules_by_component_type[mod.component_type] = mod
 
     def registerHandler(self, category, name, method):
         cat_dict = self.handlers.get(category, {})
@@ -228,6 +231,65 @@ class ModuleRegistry(object):
     def getHandler(self, category, name):
         return self.handlers[category][name]
 
+    def dispatch(self, component_type,
+                 parser, xml_parent,
+                 component, template_data={}):
+        """This is a method that you can call from your implementation of
+        Base.gen_xml or component.  It allows modules to define a type
+        of component, and benefit from extensibility via Python
+        entry points and Jenkins Job Builder :ref:`Macros <macro>`.
+
+        :arg string component_type: the name of the component
+          (e.g., `builder`)
+        :arg YAMLParser parser: the global YMAL Parser
+        :arg Element xml_parent: the parent XML element
+        :arg dict template_data: values that should be interpolated into
+          the component definition
+
+        See :py:class:`jenkins_jobs.modules.base.Base` for how to register
+        components of a module.
+
+        See the Publishers module for a simple example of how to use
+        this method.
+        """
+
+        if component_type not in self.modules_by_component_type:
+            raise JenkinsJobsException("Unknown component type: "
+                                       "'{0}'.".format(component_type))
+
+        component_list_type = self.modules_by_component_type[component_type] \
+            .component_list_type
+
+        if isinstance(component, dict):
+            # The component is a sigleton dictionary of name: dict(args)
+            name, component_data = component.items()[0]
+            if template_data:
+                # Template data contains values that should be interpolated
+                # into the component definition
+                s = yaml.dump(component_data, default_flow_style=False)
+                s = s.format(**template_data)
+                component_data = yaml.load(s)
+        else:
+            # The component is a simple string name, eg "run-tests"
+            name = component
+            component_data = {}
+
+        # Look for a component function defined in an entry point
+        for ep in pkg_resources.iter_entry_points(
+            group='jenkins_jobs.{0}'.format(component_list_type), name=name):
+            func = ep.load()
+            func(parser, xml_parent, component_data)
+        else:
+            # Otherwise, see if it's defined as a macro
+            component = parser.data.get(component_type, {}).get(name)
+            if component:
+                for b in component[component_list_type]:
+                    # Pass component_data in as template data to this function
+                    # so that if the macro is invoked with arguments,
+                    # the arguments are interpolated into the real defn.
+                    self.dispatch(component_type,
+                                  parser, xml_parent, b, component_data)
+
 
 class XmlJob(object):
     def __init__(self, xml, name):

jenkins_jobs/modules/base.py

@@ -14,8 +14,6 @@
 
 # Base class for a jenkins_jobs module
 
-import pkg_resources
-import yaml
 import xml.etree.ElementTree as XML
 
 
@@ -42,6 +40,20 @@ class Base(object):
     #: ordered XML output.
     sequence = 10
 
+    #: The component type for components of this module.  This will be
+    #: used to look for macros (they are defined singularly, and should
+    #: not be plural).
+    #: Set both component_type and component_list_type to None if module
+    #: doesn't have components.
+    component_type = None
+
+    #: The component list type will be used to look up possible
+    #: implementations of the component type via entry points (entry
+    #: points provide a list of components, so it should be plural).
+    #: Set both component_type and component_list_type to None if module
+    #: doesn't have components.
+    component_list_type = None
+
     def __init__(self, registry):
         self.registry = registry
 
@@ -70,61 +82,3 @@ class Base(object):
         """
 
         pass
-
-    def _dispatch(self, component_type, component_list_type,
-                  parser, xml_parent,
-                  component, template_data={}):
-        """This is a private helper method that you can call from your
-        implementation of gen_xml.  It allows your module to define a
-        type of component, and benefit from extensibility via Python
-        entry points and Jenkins Job Builder :ref:`Macros <macro>`.
-
-        :arg string component_type: the name of the component
-          (e.g., `builder`)
-        :arg string component_list_type: the plural name of the component
-          type (e.g., `builders`)
-        :arg YAMLParser parser: the global YMAL Parser
-        :arg Element xml_parent: the parent XML element
-        :arg dict template_data: values that should be interpolated into
-          the component definition
-
-        The value of `component_list_type` will be used to look up
-        possible implementations of the component type via entry
-        points (entry points provide a list of components, so it
-        should be plural) while `component_type` will be used to look
-        for macros (they are defined singularly, and should not be
-        plural).
-
-        See the Publishers module for a simple example of how to use
-        this method.
-        """
-
-        if isinstance(component, dict):
-            # The component is a sigleton dictionary of name: dict(args)
-            name, component_data = component.items()[0]
-            if template_data:
-                # Template data contains values that should be interpolated
-                # into the component definition
-                s = yaml.dump(component_data, default_flow_style=False)
-                s = s.format(**template_data)
-                component_data = yaml.load(s)
-        else:
-            # The component is a simple string name, eg "run-tests"
-            name = component
-            component_data = {}
-
-        # Look for a component function defined in an entry point
-        for ep in pkg_resources.iter_entry_points(
-            group='jenkins_jobs.{0}'.format(component_list_type), name=name):
-            func = ep.load()
-            func(parser, xml_parent, component_data)
-        else:
-            # Otherwise, see if it's defined as a macro
-            component = parser.data.get(component_type, {}).get(name)
-            if component:
-                for b in component[component_list_type]:
-                    # Pass component_data in as template data to this function
-                    # so that if the macro is invoked with arguments,
-                    # the arguments are interpolated into the real defn.
-                    self._dispatch(component_type, component_list_type,
-                                   parser, xml_parent, b, component_data)

jenkins_jobs/modules/builders.py

@@ -617,14 +617,17 @@ def multijob(parser, xml_parent, data):
 class Builders(jenkins_jobs.modules.base.Base):
     sequence = 60
 
+    component_type = 'builder'
+    component_list_type = 'builders'
+
     def gen_xml(self, parser, xml_parent, data):
 
         for alias in ['prebuilders', 'builders', 'postbuilders']:
             if alias in data:
                 builders = XML.SubElement(xml_parent, alias)
                 for builder in data[alias]:
-                    self._dispatch('builder', 'builders',
+                    self.registry.dispatch('builder', parser, builders,
-                                   parser, builders, builder)
+                                           builder)
 
         # Make sure freestyle projects always have a <builders> entry
         # or Jenkins v1.472 (at least) will NPE.

jenkins_jobs/modules/notifications.py

@@ -61,6 +61,9 @@ def http_endpoint(parser, xml_parent, data):
 class Notifications(jenkins_jobs.modules.base.Base):
     sequence = 22
 
+    component_type = 'notification'
+    component_list_type = 'notifications'
+
     def gen_xml(self, parser, xml_parent, data):
         properties = xml_parent.find('properties')
         if properties is None:
@@ -75,5 +78,5 @@ class Notifications(jenkins_jobs.modules.base.Base):
             endpoints_element = XML.SubElement(notify_element, 'endpoints')
 
             for endpoint in notifications:
-                self._dispatch('notification', 'notifications',
+                self.registry.dispatch('notification',
                                        parser, endpoints_element, endpoint)

jenkins_jobs/modules/parameters.py

@@ -264,6 +264,9 @@ def svn_tags_param(parser, xml_parent, data):
 class Parameters(jenkins_jobs.modules.base.Base):
     sequence = 21
 
+    component_type = 'parameter'
+    component_list_type = 'parameters'
+
     def gen_xml(self, parser, xml_parent, data):
         properties = xml_parent.find('properties')
         if properties is None:
@@ -275,5 +278,5 @@ class Parameters(jenkins_jobs.modules.base.Base):
                                    'hudson.model.ParametersDefinitionProperty')
             pdefs = XML.SubElement(pdefp, 'parameterDefinitions')
             for param in parameters:
-                self._dispatch('parameter', 'parameters',
+                self.registry.dispatch('parameter',
                                        parser, pdefs, param)

jenkins_jobs/modules/properties.py

@@ -305,11 +305,13 @@ def extended_choice(parser, xml_parent, data):
 class Properties(jenkins_jobs.modules.base.Base):
     sequence = 20
 
+    component_type = 'property'
+    component_list_type = 'properties'
+
     def gen_xml(self, parser, xml_parent, data):
         properties = xml_parent.find('properties')
         if properties is None:
             properties = XML.SubElement(xml_parent, 'properties')
 
         for prop in data.get('properties', []):
-            self._dispatch('property', 'properties',
+            self.registry.dispatch('property', parser, properties, prop)
-                           parser, properties, prop)

jenkins_jobs/modules/publishers.py

@@ -1313,9 +1313,11 @@ def join_trigger(parser, xml_parent, data):
 class Publishers(jenkins_jobs.modules.base.Base):
     sequence = 70
 
+    component_type = 'publisher'
+    component_list_type = 'publishers'
+
     def gen_xml(self, parser, xml_parent, data):
         publishers = XML.SubElement(xml_parent, 'publishers')
 
         for action in data.get('publishers', []):
-            self._dispatch('publisher', 'publishers',
+            self.registry.dispatch('publisher', parser, publishers, action)
-                           parser, publishers, action)

jenkins_jobs/modules/reporters.py

@@ -71,6 +71,9 @@ def email(parser, xml_parent, data):
 class Reporters(jenkins_jobs.modules.base.Base):
     sequence = 55
 
+    component_type = 'reporter'
+    component_list_type = 'reporters'
+
     def gen_xml(self, parser, xml_parent, data):
         if 'reporters' not in data:
             return
@@ -81,5 +84,4 @@ class Reporters(jenkins_jobs.modules.base.Base):
         reporters = XML.SubElement(xml_parent, 'reporters')
 
         for action in data.get('reporters', []):
-            self._dispatch('reporter', 'reporters',
+            self.registry.dispatch('reporter', parser, reporters, action)
-                           parser, reporters, action)

jenkins_jobs/modules/scm.py

@@ -236,6 +236,9 @@ def svn(self, xml_parent, data):
 class SCM(jenkins_jobs.modules.base.Base):
     sequence = 30
 
+    component_type = 'scm'
+    component_list_type = 'scm'
+
     def gen_xml(self, parser, xml_parent, data):
         scms = data.get('scm', [])
         if scms:
@@ -245,7 +248,6 @@ class SCM(jenkins_jobs.modules.base.Base):
                 xml_parent = XML.SubElement(xml_parent, 'scm', xml_attribs)
                 xml_parent = XML.SubElement(xml_parent, 'scms')
             for scm in data.get('scm', []):
-                self._dispatch('scm', 'scm',
+                self.registry.dispatch('scm', parser, xml_parent, scm)
-                               parser, xml_parent, scm)
         else:
             XML.SubElement(xml_parent, 'scm', {'class': 'hudson.scm.NullSCM'})

jenkins_jobs/modules/triggers.py

@@ -335,6 +335,9 @@ def github_pull_request(parser, xml_parent, data):
 class Triggers(jenkins_jobs.modules.base.Base):
     sequence = 50
 
+    component_type = 'trigger'
+    component_list_type = 'triggers'
+
     def gen_xml(self, parser, xml_parent, data):
         triggers = data.get('triggers', [])
         if not triggers:
@@ -342,5 +345,4 @@ class Triggers(jenkins_jobs.modules.base.Base):
 
         trig_e = XML.SubElement(xml_parent, 'triggers', {'class': 'vector'})
         for trigger in triggers:
-            self._dispatch('trigger', 'triggers',
+            self.registry.dispatch('trigger', parser, trig_e, trigger)
-                           parser, trig_e, trigger)

jenkins_jobs/modules/wrappers.py

@@ -350,9 +350,11 @@ def jclouds(parser, xml_parent, data):
 class Wrappers(jenkins_jobs.modules.base.Base):
     sequence = 80
 
+    component_type = 'wrapper'
+    component_list_type = 'wrappers'
+
     def gen_xml(self, parser, xml_parent, data):
         wrappers = XML.SubElement(xml_parent, 'buildWrappers')
 
         for wrap in data.get('wrappers', []):
-            self._dispatch('wrapper', 'wrappers',
+            self.registry.dispatch('wrapper', parser, wrappers, wrap)
-                           parser, wrappers, wrap)