Improve logger output for expanding templates

Output the variable inputs used that trigger an error when expanding
template names to the error logger channel in a sensible format.

Ensures that when indented variable inputs for templates result in
exceptions when expanding a template name, that the project, template
name and variables that failed to be iterated over are outputted in a
log error message along with the original set of inputs from the
project definition to make it easier for end users to find where the
error has been made in a JJB definition.

Add code to allow dumping of variables stored in OrderedDict
transparently to match the input format used in JJB definitions and
hide the implementation detail of using OrderedDict to be within the
localyaml library.

Change-Id: I660bb0ca3b109e1a861948d6a867f185047b90ae

jenkins_jobs/local_yaml.py

@@ -100,6 +100,7 @@ import re
 
 import yaml
 from yaml.constructor import BaseConstructor
+from yaml.representer import BaseRepresenter
 from yaml import YAMLObject
 
 from collections import OrderedDict
@@ -145,6 +146,14 @@ class OrderedConstructor(BaseConstructor):
         data.update(mapping)
 
 
+class OrderedRepresenter(BaseRepresenter):
+
+    def represent_yaml_mapping(self, mapping, flow_style=None):
+        tag = u'tag:yaml.org,2002:map'
+        node = self.represent_mapping(tag, mapping, flow_style=flow_style)
+        return node
+
+
 class LocalAnchorLoader(yaml.Loader):
     """Subclass for yaml.Loader which keeps Alias between calls"""
     anchors = {}
@@ -230,9 +239,23 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader):
         return re.sub(r'({|})', r'\1\1', data)
 
 
+class LocalDumper(OrderedRepresenter, yaml.Dumper):
+    def __init__(self, *args, **kwargs):
+        super(LocalDumper, self).__init__(*args, **kwargs)
+
+        # representer to ensure conversion back looks like normal
+        # mapping and hides that we use OrderedDict internally
+        self.add_representer(OrderedDict,
+                             type(self).represent_yaml_mapping)
+        # convert any tuples to lists as the JJB input is generally
+        # in list format
+        self.add_representer(tuple,
+                             type(self).represent_list)
+
+
 class BaseYAMLObject(YAMLObject):
     yaml_loader = LocalLoader
-    yaml_dumper = yaml.Dumper
+    yaml_dumper = LocalDumper
 
 
 class YamlInclude(BaseYAMLObject):
@@ -323,3 +346,7 @@ class YamlIncludeRawEscapeDeprecated(DeprecatedTag):
 def load(stream, **kwargs):
     LocalAnchorLoader.reset_anchors()
     return yaml.load(stream, functools.partial(LocalLoader, **kwargs))
+
+
+def dump(data, stream=None, **kwargs):
+    return yaml.dump(data, stream, Dumper=LocalDumper, **kwargs)

jenkins_jobs/parser.py

@@ -279,8 +279,7 @@ class YamlParser(object):
                             continue
                         template = self._getJobTemplate(group_jobname)
                         # Allow a group to override parameters set by a project
-                        d = {}
-                        d.update(project)
+                        d = type(project)(project)
                         d.update(jobparams)
                         d.update(group)
                         d.update(group_jobparams)
@@ -293,8 +292,7 @@ class YamlParser(object):
                 # see if it's a template
                 template = self._getJobTemplate(jobname)
                 if template:
-                    d = {}
-                    d.update(project)
+                    d = type(project)(project)
                     d.update(jobparams)
                     self._expandYamlForTemplateJob(d, template, jobs_glob)
                 else:
@@ -334,14 +332,29 @@ class YamlParser(object):
             params = copy.deepcopy(project)
             params = self._applyDefaults(params, template)
 
-            expanded_values = {}
-            for (k, v) in values:
-                if isinstance(v, dict):
-                    inner_key = next(iter(v))
-                    expanded_values[k] = inner_key
-                    expanded_values.update(v[inner_key])
-                else:
-                    expanded_values[k] = v
+            try:
+                expanded_values = {}
+                for (k, v) in values:
+                    if isinstance(v, dict):
+                        inner_key = next(iter(v))
+                        expanded_values[k] = inner_key
+                        expanded_values.update(v[inner_key])
+                    else:
+                        expanded_values[k] = v
+            except TypeError:
+                project_name = project.pop('name')
+                logger.error(
+                    "Exception thrown while expanding template '%s' for "
+                    "project '%s', with expansion arguments of:\n%s\n"
+                    "Original project input variables for template:\n%s\n"
+                    "Most likely the inputs have items indented incorrectly "
+                    "to describe how they should be applied.\n\nNote yaml "
+                    "'null' is mapped to python's 'None'", template_name,
+                    project_name,
+                    "".join(local_yaml.dump({k: v}, default_flow_style=False)
+                            for (k, v) in values),
+                    local_yaml.dump(project, default_flow_style=False))
+                raise
 
             params.update(expanded_values)
             params = deep_format(params, params)

tests/base.py

@@ -110,7 +110,7 @@ class BaseTestCase(testtools.TestCase):
     def setUp(self):
 
         super(BaseTestCase, self).setUp()
-        self.useFixture(fixtures.FakeLogger(level=logging.DEBUG))
+        self.logger = self.useFixture(fixtures.FakeLogger(level=logging.DEBUG))
 
     def _read_utf8_content(self):
         # if None assume empty file

tests/yamlparser/exceptions/incorrect_template_dimensions.yaml

@@ -0,0 +1,16 @@
+- project:
+    name: template_incorrect_args
+    os:
+      - ubuntu
+      - jessie
+    stream:
+      - current:
+        branch: current
+      - master:
+          branch: master
+    jobs:
+      - 'template-incorrect-args-{stream}-{os}'
+
+- job-template:
+    name: 'template-incorrect-args-{stream}-{os}'
+    disabled: true

tests/yamlparser/test_yamlparser.py

@@ -17,9 +17,30 @@
 
 import os
 
+from jenkins_jobs import parser
+from jenkins_jobs import registry
+
 from tests import base
 
 
 class TestCaseModuleYamlInclude(base.SingleJobTestCase):
     fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
     scenarios = base.get_scenarios(fixtures_path)
+
+
+class TestYamlParserExceptions(base.BaseTestCase):
+    fixtures_path = os.path.join(os.path.dirname(__file__), 'exceptions')
+
+    def test_incorrect_template_dimensions(self):
+        self.conf_filename = None
+        config = self._get_config()
+
+        yp = parser.YamlParser(config)
+        yp.parse(os.path.join(self.fixtures_path,
+                              "incorrect_template_dimensions.yaml"))
+
+        reg = registry.ModuleRegistry(config)
+
+        e = self.assertRaises(Exception, yp.expandYaml, reg)
+        self.assertIn("'NoneType' object is not iterable", str(e))
+        self.assertIn("- branch: current\n  current: null", self.logger.output)