Merge "Revert "Move macro expansion into YamlParser.""

jenkins_jobs/local_yaml.py

@@ -143,7 +143,6 @@ Examples:
         .. literalinclude:: /../../tests/yamlparser/fixtures/jinja01.yaml.inc
 """
 
-import copy
 import functools
 import io
 import logging
@@ -291,32 +290,6 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader):
     def _escape(self, data):
         return re.sub(r'({|})', r'\1\1', data)
 
-    def __deepcopy__(self, memo):
-        """
-        Make a deep copy of a LocalLoader excluding the uncopyable self.stream.
-
-        This is achieved by performing a shallow copy of self, setting the
-        stream attribute to None and then performing a deep copy of the shallow
-        copy.
-
-        (As this method will be called again on that deep copy, we also set a
-        sentinel attribute on the shallow copy to ensure that we don't recurse
-        infinitely.)
-        """
-        assert self.done, 'Unsafe to copy an in-progress loader'
-        if getattr(self, '_copy', False):
-            # This is a shallow copy for an in-progress deep copy, remove the
-            # _copy marker and return self
-            del self._copy
-            return self
-        # Make a shallow copy
-        shallow = copy.copy(self)
-        shallow.stream = None
-        shallow._copy = True
-        deep = copy.deepcopy(shallow, memo)
-        memo[id(self)] = deep
-        return deep
-
 
 class LocalDumper(OrderedRepresenter, yaml.Dumper):
     def __init__(self, *args, **kwargs):
@@ -467,12 +440,11 @@ class CustomLoader(object):
 class Jinja2Loader(CustomLoader):
     """A loader for Jinja2-templated files."""
     def __init__(self, contents):
-        self._contents = contents
+        self._template = jinja2.Template(contents)
+        self._template.environment.undefined = jinja2.StrictUndefined
 
     def format(self, **kwargs):
-        _template = jinja2.Template(self._contents)
+        return self._template.render(kwargs)
-        _template.environment.undefined = jinja2.StrictUndefined
-        return _template.render(kwargs)
 
 
 class CustomLoaderCollection(object):

jenkins_jobs/parser.py

@@ -25,7 +25,6 @@ import os
 from jenkins_jobs.constants import MAGIC_MANAGE_STRING
 from jenkins_jobs.errors import JenkinsJobsException
 from jenkins_jobs.formatter import deep_format
-from jenkins_jobs.registry import MacroRegistry
 import jenkins_jobs.local_yaml as local_yaml
 from jenkins_jobs import utils
 
@@ -82,8 +81,6 @@ class YamlParser(object):
         self.keep_desc = jjb_config.yamlparser['keep_descriptions']
         self.path = jjb_config.yamlparser['include_path']
 
-        self._macro_registry = MacroRegistry()
-
     def load_files(self, fn):
 
         # handle deprecated behavior, and check that it's not a file like
@@ -221,11 +218,6 @@ class YamlParser(object):
             job["description"] = description + \
                 self._get_managed_string().lstrip()
 
-    def _register_macros(self):
-        for component_type in self._macro_registry.component_types:
-            for macro in self.data.get(component_type, {}).values():
-                self._macro_registry.register(component_type, macro)
-
     def _getfullname(self, data):
         if 'folder' in data:
             return "%s/%s" % (data['folder'], data['name'])
@@ -241,13 +233,10 @@ class YamlParser(object):
                     if module.handle_data(self.data):
                         changed = True
 
-        self._register_macros()
-        for default in self.data.get('defaults', {}).values():
-            self._macro_registry.expand_macros(default)
         for job in self.data.get('job', {}).values():
-            self._macro_registry.expand_macros(job)
             job = self._applyDefaults(job)
             job['name'] = self._getfullname(job)
+
             if jobs_glob and not matches(job['name'], jobs_glob):
                 logger.debug("Ignoring job {0}".format(job['name']))
                 continue
@@ -410,7 +399,6 @@ class YamlParser(object):
                 raise
             expanded['name'] = self._getfullname(expanded)
 
-            self._macro_registry.expand_macros(expanded, params)
             job_name = expanded.get('name')
             if jobs_glob and not matches(job_name, jobs_glob):
                 continue

jenkins_jobs/registry.py

@@ -15,7 +15,6 @@
 
 # Manage Jenkins plugin module registry.
 
-import copy
 import logging
 import operator
 import pkg_resources
@@ -32,223 +31,6 @@ __all__ = [
 logger = logging.getLogger(__name__)
 
 
-class MacroRegistry(object):
-
-    _component_to_component_list_mapping = {}
-    _component_list_to_component_mapping = {}
-    _macros_by_component_type = {}
-    _macros_by_component_list_type = {}
-
-    def __init__(self):
-
-        for entrypoint in pkg_resources.iter_entry_points(
-                group='jenkins_jobs.macros'):
-            Mod = entrypoint.load()
-            self._component_list_to_component_mapping[
-                Mod.component_list_type] = Mod.component_type
-            self._component_to_component_list_mapping[
-                Mod.component_type] = Mod.component_list_type
-            self._macros_by_component_type[
-                Mod.component_type] = {}
-            self._macros_by_component_list_type[
-                Mod.component_list_type] = {}
-
-        self._mask_warned = {}
-
-    @property
-    def _nonempty_component_list_types(self):
-        return [clt for clt in self._macros_by_component_list_type
-                if len(self._macros_by_component_list_type[clt]) != 0]
-
-    @property
-    def component_types(self):
-        return self._macros_by_component_type.keys()
-
-    def _is_macro(self, component_name, component_list_type):
-        return (component_name in
-                self._macros_by_component_list_type[component_list_type])
-
-    def register(self, component_type, macro):
-        macro_name = macro["name"]
-        clt = self._component_to_component_list_mapping[component_type]
-        self._macros_by_component_type[component_type][macro_name] = macro
-        self._macros_by_component_list_type[clt][macro_name] = macro
-
-    def expand_macros(self, jobish, template_data=None):
-        """Create a copy of the given job-like thing, expand macros in place on
-        the copy, and return that object to calling context.
-
-        :arg dict jobish: A job-like JJB data structure. Could be anything that
-        might provide JJB "components" that get expanded to XML configuration.
-        This includes "job", "job-template", and "default" DSL items. This
-        argument is not modified in place, but rather copied so that the copy
-        may be returned to calling context.
-
-        :arg dict template_data: If jobish is a job-template, use the same
-        template data used to fill in job-template variables to fill in macro
-        variables.
-        """
-        for component_list_type in self._nonempty_component_list_types:
-            self._expand_macros_for_component_list_type(
-                jobish, component_list_type, template_data)
-
-    def _expand_macros_for_component_list_type(self,
-                                               jobish,
-                                               component_list_type,
-                                               template_data=None):
-        """In-place expansion of macros on jobish.
-
-        :arg dict jobish: A job-like JJB data structure. Could be anything that
-        might provide JJB "components" that get expanded to XML configuration.
-        This includes "job", "job-template", and "default" DSL items. This
-        argument is not modified in place, but rather copied so that the copy
-        may be returned to calling context.
-
-        :arg str component_list_type: A string value indicating which type of
-        component we are expanding macros for.
-
-        :arg dict template_data: If jobish is a job-template, use the same
-        template data used to fill in job-template variables to fill in macro
-        variables.
-        """
-        if (jobish.get("project-type", None) == "pipeline"
-                and component_list_type == "scm"):
-            # Pipeline projects have an atypical scm type, eg:
-            #
-            # - job:
-            #   name: whatever
-            #   project-type: pipeline
-            #   pipeline-scm:
-            #     script-path: nonstandard-scriptpath.groovy
-            #     scm:
-            #       - macro_name
-            #
-            # as opposed to the more typical:
-            #
-            # - job:
-            #   name: whatever2
-            #   scm:
-            #     - macro_name
-            #
-            # So we treat that case specially here.
-            component_list = jobish.get("pipeline-scm", {}).get("scm", [])
-        else:
-            component_list = jobish.get(component_list_type, [])
-
-        component_substitutions = []
-        for component in component_list:
-            macro_component_list = self._maybe_expand_macro(
-                component, component_list_type, template_data)
-
-            if macro_component_list is not None:
-                # Since macros can contain other macros, we need to recurse
-                # into the newly-expanded macro component list to expand any
-                # macros that might be hiding in there. In order to do this we
-                # have to make the macro component list look like a job by
-                # embedding it in a dictionary like so.
-                self._expand_macros_for_component_list_type(
-                    {component_list_type: macro_component_list},
-                    component_list_type,
-                    template_data)
-
-                component_substitutions.append(
-                    (component, macro_component_list))
-
-        for component, macro_component_list in component_substitutions:
-            component_index = component_list.index(component)
-            component_list.remove(component)
-            i = 0
-            for macro_component in macro_component_list:
-                component_list.insert(component_index + i, macro_component)
-                i += 1
-
-    def _maybe_expand_macro(self,
-                            component,
-                            component_list_type,
-                            template_data=None):
-        """For a given component, if it refers to a macro, return the
-        components defined for that macro with template variables (if any)
-        interpolated in.
-
-        :arg str component_list_type: A string value indicating which type of
-        component we are expanding macros for.
-
-        :arg dict template_data: If component is a macro and contains template
-        variables, use the same template data used to fill in job-template
-        variables to fill in macro variables.
-        """
-        component_copy = copy.deepcopy(component)
-
-        if isinstance(component, dict):
-            # The component is a singleton dictionary of name:
-            # dict(args)
-            component_name, component_data = next(iter(component_copy.items()))
-        else:
-            # The component is a simple string name, eg "run-tests".
-            component_name, component_data = component_copy, None
-
-        if template_data:
-            # Address the case where a macro name contains a variable to be
-            # interpolated by template variables.
-            component_name = deep_format(component_name, template_data, True)
-
-        # Check that the component under consideration actually is a
-        # macro.
-        if not self._is_macro(component_name, component_list_type):
-            return None
-
-        # Warn if the macro shadows an actual module type name for this
-        # component list type.
-        if ModuleRegistry.is_module_name(component_name, component_list_type):
-            self._mask_warned[component_name] = True
-            logger.warning(
-                "You have a macro ('%s') defined for '%s' "
-                "component list type that is masking an inbuilt "
-                "definition" % (component_name, component_list_type))
-
-        macro_component_list = self._get_macro_components(component_name,
-                                                          component_list_type)
-
-        # If macro instance contains component_data, interpolate that
-        # into macro components.
-        if component_data:
-
-            # Also use template_data, but prefer data obtained directly from
-            # the macro instance.
-            if template_data:
-                template_data = copy.deepcopy(template_data)
-                template_data.update(component_data)
-
-                macro_component_list = deep_format(
-                    macro_component_list, template_data, False)
-            else:
-                macro_component_list = deep_format(
-                    macro_component_list, component_data, False)
-
-        return macro_component_list
-
-    def _get_macro_components(self, macro_name, component_list_type):
-        """Return the list of components that a macro expands into. For example:
-
-           - wrapper:
-               name: timeout-wrapper
-               wrappers:
-                 - timeout:
-                     fail: true
-                     elastic-percentage: 150
-                     elastic-default-timeout: 90
-                     type: elastic
-
-        Provides a single "wrapper" type (corresponding to the "wrappers" list
-        type) component named "timeout" with the values shown above.
-
-        The macro_name argument in this case would be "timeout-wrapper".
-        """
-        macro_component_list = self._macros_by_component_list_type[
-            component_list_type][macro_name][component_list_type]
-        return copy.deepcopy(macro_component_list)
-
-
 class ModuleRegistry(object):
     _entry_points_cache = {}
 
@@ -347,7 +129,8 @@ class ModuleRegistry(object):
     def set_parser_data(self, parser_data):
         self.__parser_data = parser_data
 
-    def dispatch(self, component_type, xml_parent, component):
+    def dispatch(self, component_type, xml_parent,
+                 component, template_data={}):
         """This is a method that you can call from your implementation of
         Base.gen_xml or component.  It allows modules to define a type
         of component, and benefit from extensibility via Python
@@ -357,6 +140,8 @@ class ModuleRegistry(object):
           (e.g., `builder`)
         :arg YAMLParser parser: the global YAML Parser
         :arg Element xml_parent: the parent XML element
+        :arg dict template_data: values that should be interpolated into
+          the component definition
 
         See :py:class:`jenkins_jobs.modules.base.Base` for how to register
         components of a module.
@@ -375,6 +160,18 @@ class ModuleRegistry(object):
         if isinstance(component, dict):
             # The component is a singleton dictionary of name: dict(args)
             name, component_data = next(iter(component.items()))
+            if template_data:
+                # Template data contains values that should be interpolated
+                # into the component definition
+                try:
+                    component_data = deep_format(
+                        component_data, template_data,
+                        self.jjb_config.yamlparser['allow_empty_variables'])
+                except Exception:
+                    logging.error(
+                        "Failure formatting component ('%s') data '%s'",
+                        name, component_data)
+                    raise
         else:
             # The component is a simple string name, eg "run-tests"
             name = component
@@ -437,17 +234,25 @@ class ModuleRegistry(object):
             logger.debug("Cached entry point group %s = %s",
                          component_list_type, eps)
 
-        if name in eps:
+        # check for macro first
+        component = self.parser_data.get(component_type, {}).get(name)
+        if component:
+            if name in eps and name not in self.masked_warned:
+                self.masked_warned[name] = True
+                logger.warning(
+                    "You have a macro ('%s') defined for '%s' "
+                    "component type that is masking an inbuilt "
+                    "definition" % (name, component_type))
+
+            for b in component[component_list_type]:
+                # Pass component_data in as template data to this function
+                # so that if the macro is invoked with arguments,
+                # the arguments are interpolated into the real defn.
+                self.dispatch(component_type, xml_parent, b, component_data)
+        elif name in eps:
             func = eps[name].load()
             func(self, xml_parent, component_data)
         else:
             raise JenkinsJobsException("Unknown entry point or macro '{0}' "
                                        "for component type: '{1}'.".
                                        format(name, component_type))
-
-    @classmethod
-    def is_module_name(self, name, component_list_type):
-        eps = self._entry_points_cache.get(component_list_type)
-        if not eps:
-            return False
-        return (name in eps)

setup.cfg

@@ -87,16 +87,3 @@ jenkins_jobs.modules =
     triggers=jenkins_jobs.modules.triggers:Triggers
     wrappers=jenkins_jobs.modules.wrappers:Wrappers
     zuul=jenkins_jobs.modules.zuul:Zuul
-jenkins_jobs.macros =
-    builder=jenkins_jobs.modules.builders:Builders
-    general=jenkins_jobs.modules.general:General
-    hipchat=jenkins_jobs.modules.hipchat_notif:HipChat
-    metadata=jenkins_jobs.modules.metadata:Metadata
-    notification=jenkins_jobs.modules.notifications:Notifications
-    parameter=jenkins_jobs.modules.parameters:Parameters
-    property=jenkins_jobs.modules.properties:Properties
-    publisher=jenkins_jobs.modules.publishers:Publishers
-    reporter=jenkins_jobs.modules.reporters:Reporters
-    scm=jenkins_jobs.modules.scm:SCM
-    trigger=jenkins_jobs.modules.triggers:Triggers
-    wrapper=jenkins_jobs.modules.wrappers:Wrappers

tests/xml_config/test_xml_config.py

@@ -68,6 +68,9 @@ class TestXmlJobGeneratorExceptions(base.BaseTestCase):
 
         reg = registry.ModuleRegistry(config)
         reg.set_parser_data(yp.data)
+        job_data_list, view_data_list = yp.expandYaml(reg)
 
-        self.assertRaises(errors.JenkinsJobsException, yp.expandYaml, reg)
+        xml_generator = xml_config.XmlJobGenerator(reg)
+        self.assertRaises(Exception, xml_generator.generateXML, job_data_list)
+        self.assertIn("Failure formatting component", self.logger.output)
         self.assertIn("Problem formatting with args", self.logger.output)