Merge "Improve logger output for expanding templates"

jenkins_jobs/local_yaml.py

@@ -128,6 +128,7 @@ import re
 
 import yaml
 from yaml.constructor import BaseConstructor
+from yaml.representer import BaseRepresenter
 from yaml import YAMLObject
 
 from collections import OrderedDict
@@ -173,6 +174,14 @@ class OrderedConstructor(BaseConstructor):
         data.update(mapping)
 
 
+class OrderedRepresenter(BaseRepresenter):
+
+    def represent_yaml_mapping(self, mapping, flow_style=None):
+        tag = u'tag:yaml.org,2002:map'
+        node = self.represent_mapping(tag, mapping, flow_style=flow_style)
+        return node
+
+
 class LocalAnchorLoader(yaml.Loader):
     """Subclass for yaml.Loader which keeps Alias between calls"""
     anchors = {}
@@ -258,9 +267,23 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader):
         return re.sub(r'({|})', r'\1\1', data)
 
 
+class LocalDumper(OrderedRepresenter, yaml.Dumper):
+    def __init__(self, *args, **kwargs):
+        super(LocalDumper, self).__init__(*args, **kwargs)
+
+        # representer to ensure conversion back looks like normal
+        # mapping and hides that we use OrderedDict internally
+        self.add_representer(OrderedDict,
+                             type(self).represent_yaml_mapping)
+        # convert any tuples to lists as the JJB input is generally
+        # in list format
+        self.add_representer(tuple,
+                             type(self).represent_list)
+
+
 class BaseYAMLObject(YAMLObject):
     yaml_loader = LocalLoader
-    yaml_dumper = yaml.Dumper
+    yaml_dumper = LocalDumper
 
 
 class YamlInclude(BaseYAMLObject):
@@ -408,3 +431,7 @@ class LazyLoader(object):
 def load(stream, **kwargs):
     LocalAnchorLoader.reset_anchors()
     return yaml.load(stream, functools.partial(LocalLoader, **kwargs))
+
+
+def dump(data, stream=None, **kwargs):
+    return yaml.dump(data, stream, Dumper=LocalDumper, **kwargs)

jenkins_jobs/parser.py

@@ -279,8 +279,7 @@ class YamlParser(object):
                             continue
                         template = self._getJobTemplate(group_jobname)
                         # Allow a group to override parameters set by a project
-                        d = {}
-                        d.update(project)
+                        d = type(project)(project)
                         d.update(jobparams)
                         d.update(group)
                         d.update(group_jobparams)
@@ -293,8 +292,7 @@ class YamlParser(object):
                 # see if it's a template
                 template = self._getJobTemplate(jobname)
                 if template:
-                    d = {}
-                    d.update(project)
+                    d = type(project)(project)
                     d.update(jobparams)
                     self._expandYamlForTemplateJob(d, template, jobs_glob)
                 else:
@@ -334,14 +332,29 @@ class YamlParser(object):
             params = copy.deepcopy(project)
             params = self._applyDefaults(params, template)
 
-            expanded_values = {}
-            for (k, v) in values:
-                if isinstance(v, dict):
-                    inner_key = next(iter(v))
-                    expanded_values[k] = inner_key
-                    expanded_values.update(v[inner_key])
-                else:
-                    expanded_values[k] = v
+            try:
+                expanded_values = {}
+                for (k, v) in values:
+                    if isinstance(v, dict):
+                        inner_key = next(iter(v))
+                        expanded_values[k] = inner_key
+                        expanded_values.update(v[inner_key])
+                    else:
+                        expanded_values[k] = v
+            except TypeError:
+                project_name = project.pop('name')
+                logger.error(
+                    "Exception thrown while expanding template '%s' for "
+                    "project '%s', with expansion arguments of:\n%s\n"
+                    "Original project input variables for template:\n%s\n"
+                    "Most likely the inputs have items indented incorrectly "
+                    "to describe how they should be applied.\n\nNote yaml "
+                    "'null' is mapped to python's 'None'", template_name,
+                    project_name,
+                    "".join(local_yaml.dump({k: v}, default_flow_style=False)
+                            for (k, v) in values),
+                    local_yaml.dump(project, default_flow_style=False))
+                raise
 
             params.update(expanded_values)
             params = deep_format(params, params)

tests/base.py

@@ -110,7 +110,7 @@ class BaseTestCase(testtools.TestCase):
     def setUp(self):
 
         super(BaseTestCase, self).setUp()
-        self.useFixture(fixtures.FakeLogger(level=logging.DEBUG))
+        self.logger = self.useFixture(fixtures.FakeLogger(level=logging.DEBUG))
 
     def _read_utf8_content(self):
         # if None assume empty file

tests/yamlparser/exceptions/incorrect_template_dimensions.yaml

@@ -0,0 +1,16 @@
+- project:
+    name: template_incorrect_args
+    os:
+      - ubuntu
+      - jessie
+    stream:
+      - current:
+        branch: current
+      - master:
+          branch: master
+    jobs:
+      - 'template-incorrect-args-{stream}-{os}'
+
+- job-template:
+    name: 'template-incorrect-args-{stream}-{os}'
+    disabled: true

tests/yamlparser/test_yamlparser.py

@@ -17,9 +17,30 @@
 
 import os
 
+from jenkins_jobs import parser
+from jenkins_jobs import registry
+
 from tests import base
 
 
 class TestCaseModuleYamlInclude(base.SingleJobTestCase):
     fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
     scenarios = base.get_scenarios(fixtures_path)
+
+
+class TestYamlParserExceptions(base.BaseTestCase):
+    fixtures_path = os.path.join(os.path.dirname(__file__), 'exceptions')
+
+    def test_incorrect_template_dimensions(self):
+        self.conf_filename = None
+        config = self._get_config()
+
+        yp = parser.YamlParser(config)
+        yp.parse(os.path.join(self.fixtures_path,
+                              "incorrect_template_dimensions.yaml"))
+
+        reg = registry.ModuleRegistry(config)
+
+        e = self.assertRaises(Exception, yp.expandYaml, reg)
+        self.assertIn("'NoneType' object is not iterable", str(e))
+        self.assertIn("- branch: current\n  current: null", self.logger.output)