Browse Source

Revert "Move macro expansion into YamlParser."

This reverts commit e645ac2acf.

Change-Id: I56e8c8282669cbc9f963056f64e9caef8104b6bb
tags/2.0.0.0b3
Thanh Ha 1 year ago
parent
commit
eddb40babd
5 changed files with 40 additions and 285 deletions
  1. 3
    31
      jenkins_jobs/local_yaml.py
  2. 1
    13
      jenkins_jobs/parser.py
  3. 32
    227
      jenkins_jobs/registry.py
  4. 0
    13
      setup.cfg
  5. 4
    1
      tests/xml_config/test_xml_config.py

+ 3
- 31
jenkins_jobs/local_yaml.py View File

@@ -143,7 +143,6 @@ Examples:
143 143
         .. literalinclude:: /../../tests/yamlparser/fixtures/jinja01.yaml.inc
144 144
 """
145 145
 
146
-import copy
147 146
 import functools
148 147
 import io
149 148
 import logging
@@ -291,32 +290,6 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader):
291 290
     def _escape(self, data):
292 291
         return re.sub(r'({|})', r'\1\1', data)
293 292
 
294
-    def __deepcopy__(self, memo):
295
-        """
296
-        Make a deep copy of a LocalLoader excluding the uncopyable self.stream.
297
-
298
-        This is achieved by performing a shallow copy of self, setting the
299
-        stream attribute to None and then performing a deep copy of the shallow
300
-        copy.
301
-
302
-        (As this method will be called again on that deep copy, we also set a
303
-        sentinel attribute on the shallow copy to ensure that we don't recurse
304
-        infinitely.)
305
-        """
306
-        assert self.done, 'Unsafe to copy an in-progress loader'
307
-        if getattr(self, '_copy', False):
308
-            # This is a shallow copy for an in-progress deep copy, remove the
309
-            # _copy marker and return self
310
-            del self._copy
311
-            return self
312
-        # Make a shallow copy
313
-        shallow = copy.copy(self)
314
-        shallow.stream = None
315
-        shallow._copy = True
316
-        deep = copy.deepcopy(shallow, memo)
317
-        memo[id(self)] = deep
318
-        return deep
319
-
320 293
 
321 294
 class LocalDumper(OrderedRepresenter, yaml.Dumper):
322 295
     def __init__(self, *args, **kwargs):
@@ -467,12 +440,11 @@ class CustomLoader(object):
467 440
 class Jinja2Loader(CustomLoader):
468 441
     """A loader for Jinja2-templated files."""
469 442
     def __init__(self, contents):
470
-        self._contents = contents
443
+        self._template = jinja2.Template(contents)
444
+        self._template.environment.undefined = jinja2.StrictUndefined
471 445
 
472 446
     def format(self, **kwargs):
473
-        _template = jinja2.Template(self._contents)
474
-        _template.environment.undefined = jinja2.StrictUndefined
475
-        return _template.render(kwargs)
447
+        return self._template.render(kwargs)
476 448
 
477 449
 
478 450
 class CustomLoaderCollection(object):

+ 1
- 13
jenkins_jobs/parser.py View File

@@ -25,7 +25,6 @@ import os
25 25
 from jenkins_jobs.constants import MAGIC_MANAGE_STRING
26 26
 from jenkins_jobs.errors import JenkinsJobsException
27 27
 from jenkins_jobs.formatter import deep_format
28
-from jenkins_jobs.registry import MacroRegistry
29 28
 import jenkins_jobs.local_yaml as local_yaml
30 29
 from jenkins_jobs import utils
31 30
 
@@ -82,8 +81,6 @@ class YamlParser(object):
82 81
         self.keep_desc = jjb_config.yamlparser['keep_descriptions']
83 82
         self.path = jjb_config.yamlparser['include_path']
84 83
 
85
-        self._macro_registry = MacroRegistry()
86
-
87 84
     def load_files(self, fn):
88 85
 
89 86
         # handle deprecated behavior, and check that it's not a file like
@@ -221,11 +218,6 @@ class YamlParser(object):
221 218
             job["description"] = description + \
222 219
                 self._get_managed_string().lstrip()
223 220
 
224
-    def _register_macros(self):
225
-        for component_type in self._macro_registry.component_types:
226
-            for macro in self.data.get(component_type, {}).values():
227
-                self._macro_registry.register(component_type, macro)
228
-
229 221
     def _getfullname(self, data):
230 222
         if 'folder' in data:
231 223
             return "%s/%s" % (data['folder'], data['name'])
@@ -241,13 +233,10 @@ class YamlParser(object):
241 233
                     if module.handle_data(self.data):
242 234
                         changed = True
243 235
 
244
-        self._register_macros()
245
-        for default in self.data.get('defaults', {}).values():
246
-            self._macro_registry.expand_macros(default)
247 236
         for job in self.data.get('job', {}).values():
248
-            self._macro_registry.expand_macros(job)
249 237
             job = self._applyDefaults(job)
250 238
             job['name'] = self._getfullname(job)
239
+
251 240
             if jobs_glob and not matches(job['name'], jobs_glob):
252 241
                 logger.debug("Ignoring job {0}".format(job['name']))
253 242
                 continue
@@ -410,7 +399,6 @@ class YamlParser(object):
410 399
                 raise
411 400
             expanded['name'] = self._getfullname(expanded)
412 401
 
413
-            self._macro_registry.expand_macros(expanded, params)
414 402
             job_name = expanded.get('name')
415 403
             if jobs_glob and not matches(job_name, jobs_glob):
416 404
                 continue

+ 32
- 227
jenkins_jobs/registry.py View File

@@ -15,7 +15,6 @@
15 15
 
16 16
 # Manage Jenkins plugin module registry.
17 17
 
18
-import copy
19 18
 import logging
20 19
 import operator
21 20
 import pkg_resources
@@ -32,223 +31,6 @@ __all__ = [
32 31
 logger = logging.getLogger(__name__)
33 32
 
34 33
 
35
-class MacroRegistry(object):
36
-
37
-    _component_to_component_list_mapping = {}
38
-    _component_list_to_component_mapping = {}
39
-    _macros_by_component_type = {}
40
-    _macros_by_component_list_type = {}
41
-
42
-    def __init__(self):
43
-
44
-        for entrypoint in pkg_resources.iter_entry_points(
45
-                group='jenkins_jobs.macros'):
46
-            Mod = entrypoint.load()
47
-            self._component_list_to_component_mapping[
48
-                Mod.component_list_type] = Mod.component_type
49
-            self._component_to_component_list_mapping[
50
-                Mod.component_type] = Mod.component_list_type
51
-            self._macros_by_component_type[
52
-                Mod.component_type] = {}
53
-            self._macros_by_component_list_type[
54
-                Mod.component_list_type] = {}
55
-
56
-        self._mask_warned = {}
57
-
58
-    @property
59
-    def _nonempty_component_list_types(self):
60
-        return [clt for clt in self._macros_by_component_list_type
61
-                if len(self._macros_by_component_list_type[clt]) != 0]
62
-
63
-    @property
64
-    def component_types(self):
65
-        return self._macros_by_component_type.keys()
66
-
67
-    def _is_macro(self, component_name, component_list_type):
68
-        return (component_name in
69
-                self._macros_by_component_list_type[component_list_type])
70
-
71
-    def register(self, component_type, macro):
72
-        macro_name = macro["name"]
73
-        clt = self._component_to_component_list_mapping[component_type]
74
-        self._macros_by_component_type[component_type][macro_name] = macro
75
-        self._macros_by_component_list_type[clt][macro_name] = macro
76
-
77
-    def expand_macros(self, jobish, template_data=None):
78
-        """Create a copy of the given job-like thing, expand macros in place on
79
-        the copy, and return that object to calling context.
80
-
81
-        :arg dict jobish: A job-like JJB data structure. Could be anything that
82
-        might provide JJB "components" that get expanded to XML configuration.
83
-        This includes "job", "job-template", and "default" DSL items. This
84
-        argument is not modified in place, but rather copied so that the copy
85
-        may be returned to calling context.
86
-
87
-        :arg dict template_data: If jobish is a job-template, use the same
88
-        template data used to fill in job-template variables to fill in macro
89
-        variables.
90
-        """
91
-        for component_list_type in self._nonempty_component_list_types:
92
-            self._expand_macros_for_component_list_type(
93
-                jobish, component_list_type, template_data)
94
-
95
-    def _expand_macros_for_component_list_type(self,
96
-                                               jobish,
97
-                                               component_list_type,
98
-                                               template_data=None):
99
-        """In-place expansion of macros on jobish.
100
-
101
-        :arg dict jobish: A job-like JJB data structure. Could be anything that
102
-        might provide JJB "components" that get expanded to XML configuration.
103
-        This includes "job", "job-template", and "default" DSL items. This
104
-        argument is not modified in place, but rather copied so that the copy
105
-        may be returned to calling context.
106
-
107
-        :arg str component_list_type: A string value indicating which type of
108
-        component we are expanding macros for.
109
-
110
-        :arg dict template_data: If jobish is a job-template, use the same
111
-        template data used to fill in job-template variables to fill in macro
112
-        variables.
113
-        """
114
-        if (jobish.get("project-type", None) == "pipeline"
115
-                and component_list_type == "scm"):
116
-            # Pipeline projects have an atypical scm type, eg:
117
-            #
118
-            # - job:
119
-            #   name: whatever
120
-            #   project-type: pipeline
121
-            #   pipeline-scm:
122
-            #     script-path: nonstandard-scriptpath.groovy
123
-            #     scm:
124
-            #       - macro_name
125
-            #
126
-            # as opposed to the more typical:
127
-            #
128
-            # - job:
129
-            #   name: whatever2
130
-            #   scm:
131
-            #     - macro_name
132
-            #
133
-            # So we treat that case specially here.
134
-            component_list = jobish.get("pipeline-scm", {}).get("scm", [])
135
-        else:
136
-            component_list = jobish.get(component_list_type, [])
137
-
138
-        component_substitutions = []
139
-        for component in component_list:
140
-            macro_component_list = self._maybe_expand_macro(
141
-                component, component_list_type, template_data)
142
-
143
-            if macro_component_list is not None:
144
-                # Since macros can contain other macros, we need to recurse
145
-                # into the newly-expanded macro component list to expand any
146
-                # macros that might be hiding in there. In order to do this we
147
-                # have to make the macro component list look like a job by
148
-                # embedding it in a dictionary like so.
149
-                self._expand_macros_for_component_list_type(
150
-                    {component_list_type: macro_component_list},
151
-                    component_list_type,
152
-                    template_data)
153
-
154
-                component_substitutions.append(
155
-                    (component, macro_component_list))
156
-
157
-        for component, macro_component_list in component_substitutions:
158
-            component_index = component_list.index(component)
159
-            component_list.remove(component)
160
-            i = 0
161
-            for macro_component in macro_component_list:
162
-                component_list.insert(component_index + i, macro_component)
163
-                i += 1
164
-
165
-    def _maybe_expand_macro(self,
166
-                            component,
167
-                            component_list_type,
168
-                            template_data=None):
169
-        """For a given component, if it refers to a macro, return the
170
-        components defined for that macro with template variables (if any)
171
-        interpolated in.
172
-
173
-        :arg str component_list_type: A string value indicating which type of
174
-        component we are expanding macros for.
175
-
176
-        :arg dict template_data: If component is a macro and contains template
177
-        variables, use the same template data used to fill in job-template
178
-        variables to fill in macro variables.
179
-        """
180
-        component_copy = copy.deepcopy(component)
181
-
182
-        if isinstance(component, dict):
183
-            # The component is a singleton dictionary of name:
184
-            # dict(args)
185
-            component_name, component_data = next(iter(component_copy.items()))
186
-        else:
187
-            # The component is a simple string name, eg "run-tests".
188
-            component_name, component_data = component_copy, None
189
-
190
-        if template_data:
191
-            # Address the case where a macro name contains a variable to be
192
-            # interpolated by template variables.
193
-            component_name = deep_format(component_name, template_data, True)
194
-
195
-        # Check that the component under consideration actually is a
196
-        # macro.
197
-        if not self._is_macro(component_name, component_list_type):
198
-            return None
199
-
200
-        # Warn if the macro shadows an actual module type name for this
201
-        # component list type.
202
-        if ModuleRegistry.is_module_name(component_name, component_list_type):
203
-            self._mask_warned[component_name] = True
204
-            logger.warning(
205
-                "You have a macro ('%s') defined for '%s' "
206
-                "component list type that is masking an inbuilt "
207
-                "definition" % (component_name, component_list_type))
208
-
209
-        macro_component_list = self._get_macro_components(component_name,
210
-                                                          component_list_type)
211
-
212
-        # If macro instance contains component_data, interpolate that
213
-        # into macro components.
214
-        if component_data:
215
-
216
-            # Also use template_data, but prefer data obtained directly from
217
-            # the macro instance.
218
-            if template_data:
219
-                template_data = copy.deepcopy(template_data)
220
-                template_data.update(component_data)
221
-
222
-                macro_component_list = deep_format(
223
-                    macro_component_list, template_data, False)
224
-            else:
225
-                macro_component_list = deep_format(
226
-                    macro_component_list, component_data, False)
227
-
228
-        return macro_component_list
229
-
230
-    def _get_macro_components(self, macro_name, component_list_type):
231
-        """Return the list of components that a macro expands into. For example:
232
-
233
-           - wrapper:
234
-               name: timeout-wrapper
235
-               wrappers:
236
-                 - timeout:
237
-                     fail: true
238
-                     elastic-percentage: 150
239
-                     elastic-default-timeout: 90
240
-                     type: elastic
241
-
242
-        Provides a single "wrapper" type (corresponding to the "wrappers" list
243
-        type) component named "timeout" with the values shown above.
244
-
245
-        The macro_name argument in this case would be "timeout-wrapper".
246
-        """
247
-        macro_component_list = self._macros_by_component_list_type[
248
-            component_list_type][macro_name][component_list_type]
249
-        return copy.deepcopy(macro_component_list)
250
-
251
-
252 34
 class ModuleRegistry(object):
253 35
     _entry_points_cache = {}
254 36
 
@@ -347,7 +129,8 @@ class ModuleRegistry(object):
347 129
     def set_parser_data(self, parser_data):
348 130
         self.__parser_data = parser_data
349 131
 
350
-    def dispatch(self, component_type, xml_parent, component):
132
+    def dispatch(self, component_type, xml_parent,
133
+                 component, template_data={}):
351 134
         """This is a method that you can call from your implementation of
352 135
         Base.gen_xml or component.  It allows modules to define a type
353 136
         of component, and benefit from extensibility via Python
@@ -357,6 +140,8 @@ class ModuleRegistry(object):
357 140
           (e.g., `builder`)
358 141
         :arg YAMLParser parser: the global YAML Parser
359 142
         :arg Element xml_parent: the parent XML element
143
+        :arg dict template_data: values that should be interpolated into
144
+          the component definition
360 145
 
361 146
         See :py:class:`jenkins_jobs.modules.base.Base` for how to register
362 147
         components of a module.
@@ -375,6 +160,18 @@ class ModuleRegistry(object):
375 160
         if isinstance(component, dict):
376 161
             # The component is a singleton dictionary of name: dict(args)
377 162
             name, component_data = next(iter(component.items()))
163
+            if template_data:
164
+                # Template data contains values that should be interpolated
165
+                # into the component definition
166
+                try:
167
+                    component_data = deep_format(
168
+                        component_data, template_data,
169
+                        self.jjb_config.yamlparser['allow_empty_variables'])
170
+                except Exception:
171
+                    logging.error(
172
+                        "Failure formatting component ('%s') data '%s'",
173
+                        name, component_data)
174
+                    raise
378 175
         else:
379 176
             # The component is a simple string name, eg "run-tests"
380 177
             name = component
@@ -437,17 +234,25 @@ class ModuleRegistry(object):
437 234
             logger.debug("Cached entry point group %s = %s",
438 235
                          component_list_type, eps)
439 236
 
440
-        if name in eps:
237
+        # check for macro first
238
+        component = self.parser_data.get(component_type, {}).get(name)
239
+        if component:
240
+            if name in eps and name not in self.masked_warned:
241
+                self.masked_warned[name] = True
242
+                logger.warning(
243
+                    "You have a macro ('%s') defined for '%s' "
244
+                    "component type that is masking an inbuilt "
245
+                    "definition" % (name, component_type))
246
+
247
+            for b in component[component_list_type]:
248
+                # Pass component_data in as template data to this function
249
+                # so that if the macro is invoked with arguments,
250
+                # the arguments are interpolated into the real defn.
251
+                self.dispatch(component_type, xml_parent, b, component_data)
252
+        elif name in eps:
441 253
             func = eps[name].load()
442 254
             func(self, xml_parent, component_data)
443 255
         else:
444 256
             raise JenkinsJobsException("Unknown entry point or macro '{0}' "
445 257
                                        "for component type: '{1}'.".
446 258
                                        format(name, component_type))
447
-
448
-    @classmethod
449
-    def is_module_name(self, name, component_list_type):
450
-        eps = self._entry_points_cache.get(component_list_type)
451
-        if not eps:
452
-            return False
453
-        return (name in eps)

+ 0
- 13
setup.cfg View File

@@ -87,16 +87,3 @@ jenkins_jobs.modules =
87 87
     triggers=jenkins_jobs.modules.triggers:Triggers
88 88
     wrappers=jenkins_jobs.modules.wrappers:Wrappers
89 89
     zuul=jenkins_jobs.modules.zuul:Zuul
90
-jenkins_jobs.macros =
91
-    builder=jenkins_jobs.modules.builders:Builders
92
-    general=jenkins_jobs.modules.general:General
93
-    hipchat=jenkins_jobs.modules.hipchat_notif:HipChat
94
-    metadata=jenkins_jobs.modules.metadata:Metadata
95
-    notification=jenkins_jobs.modules.notifications:Notifications
96
-    parameter=jenkins_jobs.modules.parameters:Parameters
97
-    property=jenkins_jobs.modules.properties:Properties
98
-    publisher=jenkins_jobs.modules.publishers:Publishers
99
-    reporter=jenkins_jobs.modules.reporters:Reporters
100
-    scm=jenkins_jobs.modules.scm:SCM
101
-    trigger=jenkins_jobs.modules.triggers:Triggers
102
-    wrapper=jenkins_jobs.modules.wrappers:Wrappers

+ 4
- 1
tests/xml_config/test_xml_config.py View File

@@ -68,6 +68,9 @@ class TestXmlJobGeneratorExceptions(base.BaseTestCase):
68 68
 
69 69
         reg = registry.ModuleRegistry(config)
70 70
         reg.set_parser_data(yp.data)
71
+        job_data_list, view_data_list = yp.expandYaml(reg)
71 72
 
72
-        self.assertRaises(errors.JenkinsJobsException, yp.expandYaml, reg)
73
+        xml_generator = xml_config.XmlJobGenerator(reg)
74
+        self.assertRaises(Exception, xml_generator.generateXML, job_data_list)
75
+        self.assertIn("Failure formatting component", self.logger.output)
73 76
         self.assertIn("Problem formatting with args", self.logger.output)

Loading…
Cancel
Save