diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py index 9243714bd..b222a6441 100644 --- a/jenkins_jobs/local_yaml.py +++ b/jenkins_jobs/local_yaml.py @@ -100,6 +100,7 @@ import re import yaml from yaml.constructor import BaseConstructor +from yaml.representer import BaseRepresenter from yaml import YAMLObject from collections import OrderedDict @@ -145,6 +146,14 @@ class OrderedConstructor(BaseConstructor): data.update(mapping) +class OrderedRepresenter(BaseRepresenter): + + def represent_yaml_mapping(self, mapping, flow_style=None): + tag = u'tag:yaml.org,2002:map' + node = self.represent_mapping(tag, mapping, flow_style=flow_style) + return node + + class LocalAnchorLoader(yaml.Loader): """Subclass for yaml.Loader which keeps Alias between calls""" anchors = {} @@ -230,9 +239,23 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader): return re.sub(r'({|})', r'\1\1', data) +class LocalDumper(OrderedRepresenter, yaml.Dumper): + def __init__(self, *args, **kwargs): + super(LocalDumper, self).__init__(*args, **kwargs) + + # representer to ensure conversion back looks like normal + # mapping and hides that we use OrderedDict internally + self.add_representer(OrderedDict, + type(self).represent_yaml_mapping) + # convert any tuples to lists as the JJB input is generally + # in list format + self.add_representer(tuple, + type(self).represent_list) + + class BaseYAMLObject(YAMLObject): yaml_loader = LocalLoader - yaml_dumper = yaml.Dumper + yaml_dumper = LocalDumper class YamlInclude(BaseYAMLObject): @@ -323,3 +346,7 @@ class YamlIncludeRawEscapeDeprecated(DeprecatedTag): def load(stream, **kwargs): LocalAnchorLoader.reset_anchors() return yaml.load(stream, functools.partial(LocalLoader, **kwargs)) + + +def dump(data, stream=None, **kwargs): + return yaml.dump(data, stream, Dumper=LocalDumper, **kwargs) diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py index f39d08eff..4c5598841 100644 --- a/jenkins_jobs/parser.py +++ b/jenkins_jobs/parser.py @@ -279,8 +279,7 @@ class YamlParser(object): continue template = self._getJobTemplate(group_jobname) # Allow a group to override parameters set by a project - d = {} - d.update(project) + d = type(project)(project) d.update(jobparams) d.update(group) d.update(group_jobparams) @@ -293,8 +292,7 @@ class YamlParser(object): # see if it's a template template = self._getJobTemplate(jobname) if template: - d = {} - d.update(project) + d = type(project)(project) d.update(jobparams) self._expandYamlForTemplateJob(d, template, jobs_glob) else: @@ -334,14 +332,29 @@ class YamlParser(object): params = copy.deepcopy(project) params = self._applyDefaults(params, template) - expanded_values = {} - for (k, v) in values: - if isinstance(v, dict): - inner_key = next(iter(v)) - expanded_values[k] = inner_key - expanded_values.update(v[inner_key]) - else: - expanded_values[k] = v + try: + expanded_values = {} + for (k, v) in values: + if isinstance(v, dict): + inner_key = next(iter(v)) + expanded_values[k] = inner_key + expanded_values.update(v[inner_key]) + else: + expanded_values[k] = v + except TypeError: + project_name = project.pop('name') + logger.error( + "Exception thrown while expanding template '%s' for " + "project '%s', with expansion arguments of:\n%s\n" + "Original project input variables for template:\n%s\n" + "Most likely the inputs have items indented incorrectly " + "to describe how they should be applied.\n\nNote yaml " + "'null' is mapped to python's 'None'", template_name, + project_name, + "".join(local_yaml.dump({k: v}, default_flow_style=False) + for (k, v) in values), + local_yaml.dump(project, default_flow_style=False)) + raise params.update(expanded_values) params = deep_format(params, params) diff --git a/tests/base.py b/tests/base.py index 3647ee526..94de3fe9e 100644 --- a/tests/base.py +++ b/tests/base.py @@ -110,7 +110,7 @@ class BaseTestCase(testtools.TestCase): def setUp(self): super(BaseTestCase, self).setUp() - self.useFixture(fixtures.FakeLogger(level=logging.DEBUG)) + self.logger = self.useFixture(fixtures.FakeLogger(level=logging.DEBUG)) def _read_utf8_content(self): # if None assume empty file diff --git a/tests/yamlparser/exceptions/incorrect_template_dimensions.yaml b/tests/yamlparser/exceptions/incorrect_template_dimensions.yaml new file mode 100644 index 000000000..77dedefaa --- /dev/null +++ b/tests/yamlparser/exceptions/incorrect_template_dimensions.yaml @@ -0,0 +1,16 @@ +- project: + name: template_incorrect_args + os: + - ubuntu + - jessie + stream: + - current: + branch: current + - master: + branch: master + jobs: + - 'template-incorrect-args-{stream}-{os}' + +- job-template: + name: 'template-incorrect-args-{stream}-{os}' + disabled: true diff --git a/tests/yamlparser/test_yamlparser.py b/tests/yamlparser/test_yamlparser.py index a3bd90997..3f4859b88 100644 --- a/tests/yamlparser/test_yamlparser.py +++ b/tests/yamlparser/test_yamlparser.py @@ -17,9 +17,30 @@ import os +from jenkins_jobs import parser +from jenkins_jobs import registry + from tests import base class TestCaseModuleYamlInclude(base.SingleJobTestCase): fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures') scenarios = base.get_scenarios(fixtures_path) + + +class TestYamlParserExceptions(base.BaseTestCase): + fixtures_path = os.path.join(os.path.dirname(__file__), 'exceptions') + + def test_incorrect_template_dimensions(self): + self.conf_filename = None + config = self._get_config() + + yp = parser.YamlParser(config) + yp.parse(os.path.join(self.fixtures_path, + "incorrect_template_dimensions.yaml")) + + reg = registry.ModuleRegistry(config) + + e = self.assertRaises(Exception, yp.expandYaml, reg) + self.assertIn("'NoneType' object is not iterable", str(e)) + self.assertIn("- branch: current\n current: null", self.logger.output)