diff --git a/jenkins_jobs/cached_property.py b/jenkins_jobs/cached_property.py new file mode 100644 index 000000000..bab0fa562 --- /dev/null +++ b/jenkins_jobs/cached_property.py @@ -0,0 +1,9 @@ +from functools import lru_cache + +# cached_property was introduced in Python 3.8. +# TODO: Remove this file when support for Python 3.7 is dropped. +# Recipe from https://stackoverflow.com/a/19979379 + + +def cached_property(fn): + return property(lru_cache()(fn)) diff --git a/jenkins_jobs/cli/entry.py b/jenkins_jobs/cli/entry.py index e8081ac6d..fab669dbf 100644 --- a/jenkins_jobs/cli/entry.py +++ b/jenkins_jobs/cli/entry.py @@ -22,6 +22,7 @@ from pathlib import Path from stevedore import extension import yaml +from jenkins_jobs.errors import JenkinsJobsException from jenkins_jobs.cli.parser import create_parser from jenkins_jobs.config import JJBConfig from jenkins_jobs import utils @@ -174,7 +175,13 @@ def main(): argv = sys.argv[1:] jjb = JenkinsJobs(argv) - jjb.execute() + try: + jjb.execute() + except JenkinsJobsException as x: + print(file=sys.stderr) + for line in x.lines: + print(line, file=sys.stderr) + sys.exit(1) if __name__ == "__main__": diff --git a/jenkins_jobs/cli/subcommand/base.py b/jenkins_jobs/cli/subcommand/base.py index a85495eea..3a66cd075 100644 --- a/jenkins_jobs/cli/subcommand/base.py +++ b/jenkins_jobs/cli/subcommand/base.py @@ -43,7 +43,7 @@ def matches(name, glob_list): def filter_matching(item_list, glob_list): if not glob_list: return item_list - return [item for item in item_list if matches(item["name"], glob_list)] + return [item for item in item_list if matches(item.name, glob_list)] class BaseSubCommand(metaclass=abc.ABCMeta): diff --git a/jenkins_jobs/cli/subcommand/delete.py b/jenkins_jobs/cli/subcommand/delete.py index 9c42d33bb..53206c056 100644 --- a/jenkins_jobs/cli/subcommand/delete.py +++ b/jenkins_jobs/cli/subcommand/delete.py @@ -61,8 +61,8 @@ class DeleteSubCommand(base.JobsSubCommand): roots = self.load_roots(jjb_config, options.path) jobs = base.filter_matching(roots.generate_jobs(), options.name) views = base.filter_matching(roots.generate_views(), options.name) - job_names = [j["name"] for j in jobs] - view_names = [v["name"] for v in views] + job_names = [j.name for j in jobs] + view_names = [v.name for v in views] else: job_names = options.name view_names = options.name diff --git a/jenkins_jobs/cli/subcommand/list.py b/jenkins_jobs/cli/subcommand/list.py index 9fb133993..b3a9986b7 100644 --- a/jenkins_jobs/cli/subcommand/list.py +++ b/jenkins_jobs/cli/subcommand/list.py @@ -49,7 +49,7 @@ class ListSubCommand(base.JobsSubCommand): if path_list: roots = self.load_roots(jjb_config, path_list) jobs = base.filter_matching(roots.generate_jobs(), glob_list) - job_names = [j["name"] for j in jobs] + job_names = [j.name for j in jobs] else: jenkins = JenkinsManager(jjb_config) job_names = [ diff --git a/jenkins_jobs/defaults.py b/jenkins_jobs/defaults.py index 309a1d749..d771a5ba2 100644 --- a/jenkins_jobs/defaults.py +++ b/jenkins_jobs/defaults.py @@ -12,6 +12,9 @@ from dataclasses import dataclass +from .loc_loader import LocDict +from .position import Pos + job_contents_keys = { # Same as for macros. @@ -154,34 +157,40 @@ view_contents_keys = { def split_contents_params(data, contents_keys): - contents = {key: value for key, value in data.items() if key in contents_keys} - params = {key: value for key, value in data.items() if key not in contents_keys} + contents = data.copy_with( + {key: value for key, value in data.items() if key in contents_keys} + ) + params = data.copy_with( + {key: value for key, value in data.items() if key not in contents_keys} + ) return (contents, params) @dataclass class Defaults: name: str + pos: Pos params: dict contents: dict # Values that go to job contents. @classmethod - def add(cls, config, roots, expander, params_expander, data): - d = {**data} - name = d.pop("name") + def add(cls, config, roots, expander, params_expander, data, pos): + d = data.copy() + name = d.pop_required_loc_string("name") contents, params = split_contents_params( d, job_contents_keys | view_contents_keys ) - defaults = cls(name, params, contents) + defaults = cls(name, pos, params, contents) roots.defaults[name] = defaults @classmethod def empty(cls): - return Defaults("empty", params={}, contents={}) + return Defaults("empty", pos=None, params={}, contents={}) def merged_with_global(self, global_): return Defaults( name=f"{self.name}-merged-with-global", - params={**global_.params, **self.params}, - contents={**global_.contents, **self.contents}, + pos=self.pos, + params=LocDict.merge(global_.params, self.params), + contents=LocDict.merge(global_.contents, self.contents), ) diff --git a/jenkins_jobs/dimensions.py b/jenkins_jobs/dimensions.py index ca6182362..58060a06f 100644 --- a/jenkins_jobs/dimensions.py +++ b/jenkins_jobs/dimensions.py @@ -12,78 +12,91 @@ import itertools -from .errors import JenkinsJobsException +from .errors import Context, JenkinsJobsException +from .loc_loader import LocList, LocDict -def merge_dicts(dict_list): - result = {} - for d in dict_list: - result.update(d) - return result - - -class DimensionsExpander: - def __init__(self, context): - self._context = context - - def enum_dimensions_params(self, axes, params, defaults): - if not axes: - # No axes - instantiate one job/view. - yield {} - return - dim_values = [] - for axis in axes: - try: - value = params[axis] - except KeyError: - try: - value = defaults[axis] - except KeyError: - continue # May be, value would be received from an another axis values. - value = self._decode_axis_value(axis, value) - dim_values.append(value) - for values in itertools.product(*dim_values): - yield merge_dicts(values) - - def _decode_axis_value(self, axis, value): - if not isinstance(value, list): - yield {axis: value} - return - for item in value: - if not isinstance(item, dict): - yield {axis: item} - continue - if len(item.items()) != 1: - raise JenkinsJobsException( - f"Invalid parameter {axis!r} definition for template {self._context!r}:" - f" Expected a value or a dict with single element, but got: {item!r}" - ) - value, p = next(iter(item.items())) - yield { - axis: value, # Point axis value. - **p, # Point-specific parameters. May override asis value. - } - - def is_point_included(self, exclude_list, params): - return not any(self._match_exclude(params, el) for el in exclude_list or []) - - def _match_exclude(self, params, exclude): - if not isinstance(exclude, dict): +def _decode_axis_value(axis, value, key_pos, value_pos): + if not isinstance(value, (list, LocList)): + yield {axis: value} + return + for idx, item in enumerate(value): + if not isinstance(item, (dict, LocDict)): + d = LocDict() + if type(value) is LocList: + d.set_item(axis, item, key_pos, value.value_pos[idx]) + else: + d[axis] = item + yield d + continue + if len(item.items()) != 1: raise JenkinsJobsException( - f"Template {self._context!r}: Exclude element should be dict, but is: {exclude!r}" + f"Expected a value or a dict with single element, but got: {item!r}", + pos=value.value_pos[idx], + ctx=[Context(f"In pamareter {axis!r} definition", key_pos)], ) - if not exclude: - raise JenkinsJobsException( - f"Template {self._context!r}: Exclude element should be dict, but is empty: {exclude!r}" - ) - for axis, value in exclude.items(): + value, p = next(iter(item.items())) + yield LocDict.merge( + {axis: value}, # Point axis value. + p, # Point-specific parameters. May override axis value. + ) + + +def enum_dimensions_params(axes, params, defaults): + if not axes: + # No axes - instantiate one job/view. + yield {} + return + dim_values = [] + for axis in axes: + try: + value, key_pos, value_pos = params.item_with_pos(axis) + except KeyError: try: - v = params[axis] + value = defaults[axis] except KeyError: - raise JenkinsJobsException( - f"Template {self._context!r}: Unknown axis {axis!r} for exclude element: {exclude!r}" - ) - if value != v: - return False - # All required exclude values are matched. + continue # May be, value would be received from an another axis values. + value = list(_decode_axis_value(axis, value, key_pos, value_pos)) + dim_values.append(value) + for values in itertools.product(*dim_values): + yield LocDict.merge(*values) + + +def _match_exclude(params, exclude, pos): + if not isinstance(exclude, dict): + raise JenkinsJobsException( + f"Expected a dict, but got: {exclude!r}", + pos=pos, + ) + if not exclude: + raise JenkinsJobsException( + f"Expected a dict, but is empty: {exclude!r}", + pos=pos, + ) + for axis, value in exclude.items(): + try: + v = params[axis] + except KeyError: + raise JenkinsJobsException( + f"Unknown axis {axis!r}", + pos=pos, + ) + if value != v: + return False + # All required exclude values are matched. + return True + + +def is_point_included(exclude_list, params, key_pos=None): + if not exclude_list: return True + try: + for idx, exclude in enumerate(exclude_list): + if _match_exclude(params, exclude, exclude_list.value_pos[idx]): + return False + except JenkinsJobsException as x: + raise x.with_context( + f"In template exclude list", + pos=key_pos, + ) + return True diff --git a/jenkins_jobs/errors.py b/jenkins_jobs/errors.py index 2df9fd31d..cd46950c5 100644 --- a/jenkins_jobs/errors.py +++ b/jenkins_jobs/errors.py @@ -1,6 +1,9 @@ """Exception classes for jenkins_jobs errors""" import inspect +from dataclasses import dataclass + +from .position import Pos def is_sequence(arg): @@ -9,8 +12,53 @@ def is_sequence(arg): ) +def context_lines(message, pos): + if not pos: + return [message] + snippet_lines = [line.rstrip() for line in pos.snippet.splitlines()] + return [ + f"{pos.path}:{pos.line+1}:{pos.column+1}: {message}", + *snippet_lines, + ] + + +@dataclass +class Context: + message: str + pos: Pos + + @property + def lines(self): + return context_lines(self.message, self.pos) + + class JenkinsJobsException(Exception): - pass + def __init__(self, message, pos=None, ctx=None): + super().__init__(message) + self.pos = pos + self.ctx = ctx or [] # Context list + + @property + def message(self): + return self.args[0] + + def with_pos(self, pos): + return JenkinsJobsException(self.message, pos, self.ctx) + + def with_context(self, message, pos, ctx=None): + return JenkinsJobsException( + self.message, self.pos, [*(ctx or []), Context(message, pos), *self.ctx] + ) + + def with_ctx_list(self, ctx): + return JenkinsJobsException(self.message, self.pos, [*ctx, *self.ctx]) + + @property + def lines(self): + ctx_lines = [] + for ctx in self.ctx: + ctx_lines += ctx.lines + return [*ctx_lines, *context_lines(self.message, self.pos)] class ModuleError(JenkinsJobsException): @@ -37,7 +85,7 @@ class ModuleError(JenkinsJobsException): class InvalidAttributeError(ModuleError): - def __init__(self, attribute_name, value, valid_values=None): + def __init__(self, attribute_name, value, valid_values=None, pos=None, ctx=None): message = "'{0}' is an invalid value for attribute {1}.{2}".format( value, self.get_module_name(), attribute_name ) @@ -47,11 +95,11 @@ class InvalidAttributeError(ModuleError): ", ".join("'{0}'".format(value) for value in valid_values) ) - super(InvalidAttributeError, self).__init__(message) + super().__init__(message, pos, ctx) class MissingAttributeError(ModuleError): - def __init__(self, missing_attribute, module_name=None): + def __init__(self, missing_attribute, module_name=None, pos=None, ctx=None): module = module_name or self.get_module_name() if is_sequence(missing_attribute): message = "One of {0} must be present in '{1}'".format( @@ -62,7 +110,7 @@ class MissingAttributeError(ModuleError): missing_attribute, module ) - super(MissingAttributeError, self).__init__(message) + super().__init__(message, pos, ctx) class AttributeConflictError(ModuleError): diff --git a/jenkins_jobs/expander.py b/jenkins_jobs/expander.py index a362c97fe..3af85460f 100644 --- a/jenkins_jobs/expander.py +++ b/jenkins_jobs/expander.py @@ -14,8 +14,9 @@ from functools import partial from jinja2 import StrictUndefined -from .errors import JenkinsJobsException +from .errors import Context, JenkinsJobsException from .formatter import CustomFormatter, enum_str_format_required_params +from .loc_loader import LocDict, LocString, LocList from .yaml_objects import ( J2String, J2Yaml, @@ -27,21 +28,30 @@ from .yaml_objects import ( ) -def expand_dict(expander, obj, params): - result = {} +def expand_dict(expander, obj, params, key_pos, value_pos): + result = LocDict(pos=obj.pos) for key, value in obj.items(): - expanded_key = expander.expand(key, params) - expanded_value = expander.expand(value, params) - result[expanded_key] = expanded_value + expanded_key = expander.expand(key, params, None) + expanded_value = expander.expand( + value, params, obj.key_pos.get(key), obj.value_pos.get(key) + ) + result.set_item( + expanded_key, expanded_value, obj.key_pos.get(key), obj.value_pos.get(key) + ) return result -def expand_list(expander, obj, params): - return [expander.expand(item, params) for item in obj] +def expand_list(expander, obj, params, key_pos, value_pos): + items = [ + expander.expand(item, params, None, obj.value_pos[idx]) + for idx, item in enumerate(obj) + ] + value_pos = [obj.value_pos[idx] for idx, _ in enumerate(obj)] + return LocList(items, obj.pos, value_pos) -def expand_tuple(expander, obj, params): - return tuple(expander.expand(item, params) for item in obj) +def expand_tuple(expander, obj, params, key_pos, value_pos): + return tuple(expander.expand(item, params, None) for item in obj) class StrExpander: @@ -49,19 +59,31 @@ class StrExpander: allow_empty = config.yamlparser["allow_empty_variables"] self._formatter = CustomFormatter(allow_empty) - def __call__(self, obj, params): - return self._formatter.format(obj, **params) + def __call__(self, obj, params, key_pos, value_pos): + try: + return self._formatter.format(str(obj), **params) + except JenkinsJobsException as x: + lines = str(obj).splitlines() + start_ofs = value_pos.body.index(lines[0]) + pre_pad = value_pos.body[:start_ofs] + # Shift position to reflect template position inside yaml file: + if "\n" in pre_pad: + pos = value_pos.with_offset(line_ofs=1) + else: + pos = value_pos.with_offset(column_ofs=start_ofs) + pos = pos.with_contents_start() + raise x.with_pos(pos) -def call_expand(expander, obj, params): +def call_expand(expander, obj, params, key_pos, value_pos): return obj.expand(expander, params) -def call_subst(expander, obj, params): +def call_subst(expander, obj, params, key_pos, value_pos): return obj.subst(expander, params) -def dont_expand(obj, params): +def dont_expand(obj, params, key_pos, value_pos): return obj @@ -90,9 +112,12 @@ class Expander: } self.expanders = { dict: partial(expand_dict, self), + LocDict: partial(expand_dict, self), list: partial(expand_list, self), + LocList: partial(expand_list, self), tuple: partial(expand_tuple, self), str: dont_expand, + LocString: dont_expand, bool: dont_expand, int: dont_expand, float: dont_expand, @@ -100,13 +125,15 @@ class Expander: **_yaml_object_expanders, } - def expand(self, obj, params): + def expand(self, obj, params, key_pos=None, value_pos=None): t = type(obj) try: expander = self.expanders[t] except KeyError: - raise RuntimeError(f"Do not know how to expand type: {t!r}") - return expander(obj, params) + raise JenkinsJobsException( + f"Do not know how to expand type: {t!r}", pos=value_pos + ) + return expander(obj, params, key_pos, value_pos) # Expands string formats also. Used in jobs templates and macros with parameters. @@ -119,27 +146,33 @@ class ParamsExpander(Expander): self.expanders.update( { str: StrExpander(config), + LocString: StrExpander(config), **_yaml_object_expanders, } ) -def call_required_params(obj): +def call_required_params(obj, pos): yield from obj.required_params -def enum_dict_params(obj): +def enum_dict_params(obj, pos): for key, value in obj.items(): - yield from enum_required_params(key) - yield from enum_required_params(value) + yield from enum_required_params(key, obj.key_pos.get(key)) + yield from enum_required_params(value, obj.value_pos.get(key)) -def enum_seq_params(obj): - for value in obj: - yield from enum_required_params(value) +def enum_seq_params(obj, pos): + for idx, value in enumerate(obj): + yield from enum_required_params(value, pos=None) -def no_parameters(obj): +def enum_loc_list_params(obj, pos): + for idx, value in enumerate(obj): + yield from enum_required_params(value, obj.value_pos[idx]) + + +def no_parameters(obj, pos): return [] @@ -147,8 +180,11 @@ yaml_classes_enumers = {cls: call_required_params for cls in yaml_classes_list} param_enumers = { str: enum_str_format_required_params, + LocString: enum_str_format_required_params, dict: enum_dict_params, + LocDict: enum_dict_params, list: enum_seq_params, + LocList: enum_loc_list_params, tuple: enum_seq_params, bool: no_parameters, int: no_parameters, @@ -161,53 +197,68 @@ param_enumers = { disable_expand_for = {"template-name"} -def enum_required_params(obj): +def enum_required_params(obj, pos): t = type(obj) try: enumer = param_enumers[t] except KeyError: - raise RuntimeError( - f"Do not know how to enumerate required parameters for type: {t!r}" + raise JenkinsJobsException( + f"Do not know how to enumerate required parameters for type: {t!r}", + pos=pos, ) - return enumer(obj) + return enumer(obj, pos) -def expand_parameters(expander, param_dict, template_name): - expanded_params = {} - deps = {} # Using dict as ordered set. +def expand_parameters(expander, param_dict): + expanded_params = LocDict() + deps = {} # Variable name -> variable pos. + + def deps_context(): + return [Context(f"Used by {n}", vp) for n, (kp, vp) in deps.items()] def expand(name): try: - return expanded_params[name] + value = expanded_params[name] + key_pos = expanded_params.key_pos.get(name) + value_pos = expanded_params.value_pos.get(name) + return (value, key_pos, value_pos) except KeyError: pass try: format = param_dict[name] except KeyError: - return StrictUndefined(name=name) + return (StrictUndefined(name=name), None, None) + key_pos = param_dict.key_pos.get(name) + value_pos = param_dict.value_pos.get(name) if name in deps: - raise RuntimeError( - f"While expanding {name!r} for template {template_name!r}:" - f" Recursive parameters usage: {name} <- {' <- '.join(deps)}" + expand_ctx = Context(f"While expanding {name!r}", key_pos) + raise JenkinsJobsException( + f"Recursive parameters usage: {' <- '.join(deps)}", + pos=value_pos, + ctx=[*deps_context(), expand_ctx], ) if name in disable_expand_for: value = format else: - required_params = list(enum_required_params(format)) - deps[name] = None + required_params = list(enum_required_params(format, value_pos)) + deps[name] = (key_pos, value_pos) try: - params = {n: expand(n) for n in required_params} + params = LocDict() + for n in required_params: + v, kp, vp = expand(n) + params.set_item(n, v, kp, vp) finally: deps.popitem() try: - value = expander.expand(format, params) + value = expander.expand(format, params, key_pos, value_pos) except JenkinsJobsException as x: - used_by_deps = ", used by".join(f"{d!r}" for d in deps) - raise RuntimeError( - f"While expanding {name!r}, used by {used_by_deps}, used by template {template_name!r}: {x}" + raise x.with_context( + f"While expanding parameter {name!r}", + pos=key_pos, + ctx=deps_context(), ) - expanded_params[name] = value - return value + expanded_params.set_item(name, value, key_pos, value_pos) + return (value, key_pos, value_pos) for name in param_dict: expand(name) diff --git a/jenkins_jobs/formatter.py b/jenkins_jobs/formatter.py index 77e7ff31b..c7741c358 100644 --- a/jenkins_jobs/formatter.py +++ b/jenkins_jobs/formatter.py @@ -97,7 +97,7 @@ class CustomFormatter(Formatter): continue arg_used, rest = _string.formatter_field_name_split(field_name) if arg_used == "" or type(arg_used) is int: - raise RuntimeError( + raise JenkinsJobsException( f"Positional format arguments are not supported: {format_string!r}" ) yield arg_used @@ -121,11 +121,14 @@ class CustomFormatter(Formatter): raise JenkinsJobsException(f"Missing parameter: {key!r}") -def enum_str_format_required_params(format): +def enum_str_format_required_params(format, pos): formatter = CustomFormatter() - yield from formatter.enum_required_params(format) + try: + yield from formatter.enum_required_params(str(format)) + except JenkinsJobsException as x: + raise x.with_pos(pos) def enum_str_format_param_defaults(format): formatter = CustomFormatter() - yield from formatter.enum_param_defaults(format) + yield from formatter.enum_param_defaults(str(format)) diff --git a/jenkins_jobs/job.py b/jenkins_jobs/job.py index 781ffba70..7e5bfb3fd 100644 --- a/jenkins_jobs/job.py +++ b/jenkins_jobs/job.py @@ -12,6 +12,8 @@ from dataclasses import dataclass +from .errors import JenkinsJobsException +from .loc_loader import LocDict from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group from .defaults import split_contents_params, job_contents_keys @@ -22,15 +24,15 @@ class JobBase(RootBase): folder: str @classmethod - def from_dict(cls, config, roots, expander, data): + def from_dict(cls, config, roots, expander, data, pos): keep_descriptions = config.yamlparser["keep_descriptions"] - d = {**data} - name = d.pop("name") - id = d.pop("id", None) - description = d.pop("description", None) - defaults = d.pop("defaults", "global") - project_type = d.pop("project-type", None) - folder = d.pop("folder", None) + d = data.copy() + name = d.pop_required_loc_string("name") + id = d.pop_loc_string("id", None) + description = d.pop_loc_string("description", None) + defaults = d.pop_loc_string("defaults", "global") + project_type = d.pop_loc_string("project-type", None) + folder = d.pop_loc_string("folder", None) contents, params = split_contents_params(d, job_contents_keys) return cls( roots.defaults, @@ -38,6 +40,7 @@ class JobBase(RootBase): keep_descriptions, id, name, + pos, description, defaults, params, @@ -47,10 +50,10 @@ class JobBase(RootBase): ) def _as_dict(self): - data = { - "name": self._full_name, - **self.contents, - } + data = LocDict.merge( + {"name": self._full_name}, + self.contents, + ) if self.project_type: data["project-type"] = self.project_type return data @@ -65,17 +68,23 @@ class JobBase(RootBase): class Job(JobBase, NonTemplateRootMixin): @classmethod - def add(cls, config, roots, expander, param_expander, data): - job = cls.from_dict(config, roots, expander, data) + def add(cls, config, roots, expander, param_expander, data, pos): + job = cls.from_dict(config, roots, expander, data, pos) roots.assign(roots.jobs, job.id, job, "job") + def __str__(self): + return f"job {self.name!r}" + class JobTemplate(JobBase, TemplateRootMixin): @classmethod - def add(cls, config, roots, expander, params_expander, data): - template = cls.from_dict(config, roots, params_expander, data) + def add(cls, config, roots, expander, params_expander, data, pos): + template = cls.from_dict(config, roots, params_expander, data, pos) roots.assign(roots.job_templates, template.id, template, "job template") + def __str__(self): + return f"job template {self.name!r}" + @dataclass class JobGroup(Group): @@ -83,15 +92,16 @@ class JobGroup(Group): _job_templates: dict @classmethod - def add(cls, config, roots, expander, params_expander, data): - d = {**data} - name = d.pop("name") - job_specs = [ - cls._spec_from_dict(item, error_context=f"Job group {name}") - for item in d.pop("jobs", []) - ] + def add(cls, config, roots, expander, params_expander, data, pos): + d = data.copy() + name = d.pop_required_loc_string("name") + try: + job_specs = cls._specs_from_list(d.pop("jobs", None)) + except JenkinsJobsException as x: + raise x.with_context(f"In job {name!r}", pos=pos) group = cls( name, + pos, job_specs, d, roots.jobs, @@ -100,7 +110,7 @@ class JobGroup(Group): roots.assign(roots.job_groups, group.name, group, "job group") def __str__(self): - return f"Job group {self.name}" + return f"job group {self.name!r}" @property def _root_dicts(self): diff --git a/jenkins_jobs/loader.py b/jenkins_jobs/loader.py index 54d7efc92..d6708d2a9 100644 --- a/jenkins_jobs/loader.py +++ b/jenkins_jobs/loader.py @@ -14,9 +14,8 @@ import io import logging from functools import partial -import yaml - from .errors import JenkinsJobsException +from .loc_loader import LocLoader from .yaml_objects import BaseYamlObject from .expander import Expander, ParamsExpander, deprecated_yaml_tags, yaml_classes_list from .roots import root_adders @@ -24,13 +23,13 @@ from .roots import root_adders logger = logging.getLogger(__name__) -class Loader(yaml.Loader): +class Loader(LocLoader): @classmethod def empty(cls, jjb_config): return cls(io.StringIO(), jjb_config) def __init__(self, stream, jjb_config, source_path=None, anchors=None): - super().__init__(stream) + super().__init__(stream, source_path) self.jjb_config = jjb_config self.source_path = source_path self._retain_anchors = jjb_config.yamlparser["retain_anchors"] @@ -74,10 +73,10 @@ def load_deprecated_yaml(tag, cls, loader, node): for cls in yaml_classes_list: - yaml.add_constructor(cls.yaml_tag, cls.from_yaml, Loader) + Loader.add_constructor(cls.yaml_tag, cls.from_yaml) for tag, cls in deprecated_yaml_tags: - yaml.add_constructor(tag, partial(load_deprecated_yaml, tag, cls), Loader) + Loader.add_constructor(tag, partial(load_deprecated_yaml, tag, cls)) def is_stdin(path): @@ -122,19 +121,21 @@ def load_files(config, roots, path_list): data = loader.load_path(path) if not isinstance(data, list): raise JenkinsJobsException( - f"The topmost collection in file '{path}' must be a list," - f" not a {type(data)}" + f"The topmost collection must be a list, but is: {data}", + pos=data.pos, ) - for item in data: + for idx, item in enumerate(data): if not isinstance(item, dict): raise JenkinsJobsException( - f"{path}: Topmost list should contain single-item dict," - f" not a {type(item)}. Missing indent?" + f"Topmost list should contain single-item dict," + f" not a {type(item)}. Missing indent?", + pos=data.value_pos[idx], ) if len(item) != 1: raise JenkinsJobsException( - f"{path}: Topmost dict should be single-item," - f" but have keys {item.keys()}. Missing indent?" + f"Topmost dict should be single-item," + f" but have keys {list(item.keys())}. Missing indent?", + pos=item.pos, ) kind, contents = next(iter(item.items())) if kind.startswith("_"): @@ -145,7 +146,8 @@ def load_files(config, roots, path_list): adder = root_adders[kind] except KeyError: raise JenkinsJobsException( - f"{path}: Unknown topmost element type : {kind!r}," - f" Known are: {','.join(root_adders)}." + f"Unknown topmost element type : {kind!r};" + f" known are: {','.join(root_adders)}.", + pos=item.pos, ) - adder(config, roots, expander, params_expander, contents) + adder(config, roots, expander, params_expander, contents, item.pos) diff --git a/jenkins_jobs/loc_loader.py b/jenkins_jobs/loc_loader.py new file mode 100644 index 000000000..104df24fb --- /dev/null +++ b/jenkins_jobs/loc_loader.py @@ -0,0 +1,161 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from collections import UserString + +import yaml + +from .errors import JenkinsJobsException +from .position import Pos + + +class LocDict(dict): + """dict implementation with added source position information""" + + def __init__(self, value=None, pos=None, key_pos=None, value_pos=None): + super().__init__(value or []) + self.pos = pos + self.key_pos = key_pos or {} # key -> key pos. + self.value_pos = value_pos or {} # key -> value pos. + + def item_with_pos(self, key): + value = self[key] # KeyError is propagated from here. + key_pos = self.key_pos.get(key) + value_pos = self.value_pos.get(key) + return (value, key_pos, value_pos) + + def pop_loc_string(self, key, default_value): + value = super().pop(key, default_value) + if type(value) is str: + return LocString(value, self.value_pos.get(key)) + else: + return value + + def pop_required_loc_string(self, name): + try: + value = self.pop(name) + except KeyError: + raise JenkinsJobsException( + f"Missing required element: {name!r}", + pos=self.pos, + ) + return LocString(value, self.value_pos.get(name)) + + def pop_required_element(self, name): + try: + return self.pop(name) + except KeyError: + raise JenkinsJobsException( + f"Missing required element: {name!r}", + pos=self.pos, + ) + + def copy(self): + return LocDict(self, self.pos, self.key_pos, self.value_pos) + + def copy_with(self, value): + return LocDict(value, self.pos, self.key_pos, self.value_pos) + + def __setitem__(self, key, value): + if type(value) is LocString: + super().__setitem__(key, str(value)) + self.value_pos[key] = value.pos + else: + super().__setitem__(key, value) + + def set_item(self, key, value, key_pos, value_pos): + self[key] = value + if key_pos: + self.key_pos[key] = key_pos + if value_pos: + self.value_pos[key] = value_pos + + @classmethod + def merge(cls, *args, pos=None): + result = LocDict(pos=pos) + for d in args: + result.update(d) + if type(d) is cls: + result.key_pos.update(d.key_pos) + result.value_pos.update(d.value_pos) + return result + + def update(self, d): + super().update(d) + if type(d) is LocDict: + self.key_pos.update(d.key_pos) + self.value_pos.update(d.value_pos) + + +class LocList(list): + """list implementation with added source position information""" + + def __init__(self, value=None, pos=None, value_pos=None): + if value is None: + value = [] + super().__init__(value) + self.pos = pos + self.value_pos = value_pos or [None for _ in value] # Value pos list. + + def copy(self): + return LocList(self, self.pos, self.value_pos) + + +class LocString(UserString): + """str implementation with added source position information""" + + def __init__(self, value="", pos=None): + super().__init__(value) + self.pos = pos + + +class LocLoader(yaml.Loader): + """Load YAML and store source position information""" + + def __init__(self, stream, file_path, line_ofs=0, column_ofs=0): + super().__init__(stream) + if file_path: + # Override one set by yaml Reader. Used to construct marks. + self.name = file_path + self._line_ofs = line_ofs + self._column_ofs = column_ofs + + def pos_from_node(self, node): + return Pos.from_node(node, self._line_ofs, self._column_ofs) + + def construct_yaml_map(self, node): + data = LocDict(pos=self.pos_from_node(node)) + yield data + value = self.construct_mapping(node) + data.update(value) + data.key_pos.update( + { + key_node.value: self.pos_from_node(key_node) + for key_node, value_node in node.value + } + ) + data.value_pos.update( + { + key_node.value: self.pos_from_node(value_node) + for key_node, value_node in node.value + } + ) + + def construct_yaml_seq(self, node): + data = LocList(pos=self.pos_from_node(node)) + yield data + data.extend(self.construct_sequence(node)) + data.value_pos.extend(self.pos_from_node(item_node) for item_node in node.value) + + +LocLoader.add_constructor("tag:yaml.org,2002:map", LocLoader.construct_yaml_map) +LocLoader.add_constructor("tag:yaml.org,2002:seq", LocLoader.construct_yaml_seq) diff --git a/jenkins_jobs/macro.py b/jenkins_jobs/macro.py index 5890cc9f1..f643e35f5 100644 --- a/jenkins_jobs/macro.py +++ b/jenkins_jobs/macro.py @@ -14,6 +14,7 @@ from dataclasses import dataclass from functools import partial from .errors import JenkinsJobsException +from .position import Pos macro_specs = [ @@ -33,20 +34,31 @@ macro_specs = [ @dataclass class Macro: name: str + pos: Pos elements: list @classmethod def add( - cls, type_name, elements_name, config, roots, expander, params_expander, data + cls, + type_name, + elements_name, + config, + roots, + expander, + params_expander, + data, + pos, ): - d = {**data} - name = d.pop("name") - elements = d.pop(elements_name) + d = data.copy() + name = d.pop_required_loc_string("name") + elements = d.pop_required_element(elements_name) if d: + example_key = next(iter(d.keys())) raise JenkinsJobsException( - f"Macro {type_name} {name!r}: unexpected elements: {','.join(d.keys())}" + f"In {type_name} macro {name!r}: unexpected elements: {','.join(d.keys())}", + pos=data.key_pos.get(example_key), ) - macro = cls(name, elements or []) + macro = cls(name, pos, elements or []) roots.assign(roots.macros[type_name], name, macro, "macro") diff --git a/jenkins_jobs/modules/project_multibranch.py b/jenkins_jobs/modules/project_multibranch.py index 7a9bb4e0f..386d00175 100644 --- a/jenkins_jobs/modules/project_multibranch.py +++ b/jenkins_jobs/modules/project_multibranch.py @@ -87,7 +87,7 @@ import six from jenkins_jobs.modules.scm import git_extensions from jenkins_jobs.errors import InvalidAttributeError, MissingAttributeError -from jenkins_jobs.errors import JenkinsJobsException +from jenkins_jobs.errors import Context, JenkinsJobsException from jenkins_jobs.xml_config import remove_ignorable_whitespace logger = logging.getLogger(str(__name__)) @@ -1838,14 +1838,24 @@ def apply_property_strategies(props_elem, props_list): ) if isinstance(tbopc_val, dict): if "comment" not in tbopc_val: - raise MissingAttributeError("trigger-build-on-pr-comment[comment]") + raise MissingAttributeError( + "trigger-build-on-pr-comment[comment]", + pos=dbs_list.key_pos.get("trigger-build-on-pr-comment"), + ) XML.SubElement(tbopc_elem, "commentBody").text = tbopc_val["comment"] if tbopc_val.get("allow-untrusted-users", False): XML.SubElement(tbopc_elem, "allowUntrusted").text = "true" elif isinstance(tbopc_val, str): XML.SubElement(tbopc_elem, "commentBody").text = tbopc_val else: - raise InvalidAttributeError("trigger-build-on-pr-comment", tbopc_val) + attr = "trigger-build-on-pr-comment" + ctx = [Context(f"For attribute {attr!r}", dbs_list.key_pos.get(attr))] + raise InvalidAttributeError( + attr, + tbopc_val, + pos=dbs_list.value_pos.get(attr), + ctx=ctx, + ) for opt in pcb_bool_opts: opt_value = dbs_list.get(opt, None) if opt_value: @@ -1861,7 +1871,10 @@ def apply_property_strategies(props_elem, props_list): # no sub-elements in this case pass else: - raise InvalidAttributeError(opt, opt_value) + ctx = Context(f"For attribute {opt!r}", dbs_list.key_pos.get(opt)) + raise InvalidAttributeError( + opt, opt_value, pos=dbs_list.value_pos.get(opt), ctx=[ctx] + ) def add_filter_branch_pr_behaviors(traits, data): diff --git a/jenkins_jobs/modules/view_nested.py b/jenkins_jobs/modules/view_nested.py index 02388e5f7..77877ce9d 100644 --- a/jenkins_jobs/modules/view_nested.py +++ b/jenkins_jobs/modules/view_nested.py @@ -42,6 +42,7 @@ Example: import xml.etree.ElementTree as XML import jenkins_jobs.modules.base import jenkins_jobs.modules.helpers as helpers +from jenkins_jobs.root_base import JobViewData from jenkins_jobs.xml_config import XmlViewGenerator COLUMN_DICT = { @@ -66,9 +67,10 @@ class Nested(jenkins_jobs.modules.base.Base): v_xml = XML.SubElement(root, "views") views = data.get("views", []) + view_data_list = [JobViewData(v) for v in views] xml_view_generator = XmlViewGenerator(self.registry) - xml_views = xml_view_generator.generateXML(views) + xml_views = xml_view_generator.generateXML(view_data_list) for xml_job in xml_views: v_xml.append(xml_job.xml) diff --git a/jenkins_jobs/modules/wrappers.py b/jenkins_jobs/modules/wrappers.py index eb91a65b6..361b32adf 100644 --- a/jenkins_jobs/modules/wrappers.py +++ b/jenkins_jobs/modules/wrappers.py @@ -1828,7 +1828,7 @@ def pre_scm_buildstep(registry, xml_parent, data): xml_parent, "org.jenkinsci.plugins.preSCMbuildstep." "PreSCMBuildStepsWrapper" ) bs = XML.SubElement(bsp, "buildSteps") - stepList = data if type(data) is list else data.get("buildsteps") + stepList = data if isinstance(data, list) else data.get("buildsteps") for step in stepList: for edited_node in create_builders(registry, step): diff --git a/jenkins_jobs/position.py b/jenkins_jobs/position.py new file mode 100644 index 000000000..946b39291 --- /dev/null +++ b/jenkins_jobs/position.py @@ -0,0 +1,93 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import yaml + +if sys.version_info >= (3, 8): + from functools import cached_property +else: + from .cached_property import cached_property + + +LINE_SEPARATORS = "\0\r\n\x85\u2028\u2029" +WHITESPACE_CHARS = " \t" + + +class Pos: + @classmethod + def from_node(cls, node, line_ofs=0, column_ofs=0): + mark = node.start_mark + return cls(mark, mark.name, mark.line + line_ofs, mark.column + column_ofs) + + @classmethod + def from_file(cls, path, text): + mark = yaml.Mark(str(path), 0, 0, 0, text, 0) + return cls(mark, path, 0, 0) + + def __init__(self, mark, path, line, column): + self._mark = mark + self.path = path + self.line = line # Starts from 0. + self.column = column # Starts from 0. + + def __repr__(self): + return f"" + + def with_offset(self, line_ofs=0, column_ofs=0): + line_ptr = self._move_ptr_by_lines(line_ofs) + ptr = line_ptr + column_ofs + mark = self._clone_mark(self._mark, ptr) + if line_ofs: + column = column_ofs # Start from new line. + else: + column = self.column + column_ofs + return Pos(mark, self.path, self.line + line_ofs, column) + + def with_contents_start(self): + ptr = self._mark.pointer + buf = self._mark.buffer + while ( + ptr < len(buf) + and buf[ptr] not in LINE_SEPARATORS + and buf[ptr] in WHITESPACE_CHARS + ): + ptr += 1 + mark = self._clone_mark(self._mark, ptr) + return Pos(mark, self.path, self.line, self.column + ptr - self._mark.pointer) + + @cached_property + def snippet(self): + return self._mark.get_snippet(max_length=100) + + @cached_property + def body(self): + return self._mark.buffer[self._mark.pointer :] + + def _clone_mark(self, mark, ptr): + return yaml.Mark( + mark.name, + mark.index, + mark.line, + mark.column, + mark.buffer, + ptr, + ) + + def _move_ptr_by_lines(self, line_ofs): + ptr = self._mark.pointer + buf = self._mark.buffer + while line_ofs > 0 and ptr < len(buf): + if buf[ptr] in LINE_SEPARATORS: + line_ofs -= 1 + ptr += 1 + return ptr diff --git a/jenkins_jobs/project.py b/jenkins_jobs/project.py index 02ec2d50a..c737eb2e5 100644 --- a/jenkins_jobs/project.py +++ b/jenkins_jobs/project.py @@ -12,6 +12,8 @@ from dataclasses import dataclass +from .errors import JenkinsJobsException +from .position import Pos from .root_base import GroupBase @@ -24,24 +26,22 @@ class Project(GroupBase): _view_templates: dict _view_groups: dict name: str + pos: Pos defaults_name: str job_specs: list # list[Spec] view_specs: list # list[Spec] params: dict @classmethod - def add(cls, config, roots, expander, params_expander, data): - d = {**data} - name = d.pop("name") - defaults = d.pop("defaults", None) - job_specs = [ - cls._spec_from_dict(item, error_context=f"Project {name}") - for item in d.pop("jobs", []) - ] - view_specs = [ - cls._spec_from_dict(item, error_context=f"Project {name}") - for item in d.pop("views", []) - ] + def add(cls, config, roots, expander, params_expander, data, pos): + d = data.copy() + name = d.pop_required_loc_string("name") + defaults = d.pop_loc_string("defaults", None) + try: + job_specs = cls._specs_from_list(d.pop("jobs", None)) + view_specs = cls._specs_from_list(d.pop("views", None)) + except JenkinsJobsException as x: + raise x.with_context(f"In project {name!r}", pos=pos) project = cls( roots.jobs, roots.job_templates, @@ -50,6 +50,7 @@ class Project(GroupBase): roots.view_templates, roots.view_groups, name, + pos, defaults, job_specs, view_specs, @@ -58,7 +59,7 @@ class Project(GroupBase): roots.assign(roots.projects, project.name, project, "project") def __str__(self): - return f"Project {self.name}" + return f"project {self.name!r}" @property def _my_params(self): diff --git a/jenkins_jobs/registry.py b/jenkins_jobs/registry.py index f9ea22271..65a412a00 100644 --- a/jenkins_jobs/registry.py +++ b/jenkins_jobs/registry.py @@ -172,9 +172,9 @@ class ModuleRegistry(object): def amend_job_dicts(self, job_data_list): while True: changed = False - for data in job_data_list: + for job in job_data_list: for module in self.modules: - if module.amend_job_dict(data): + if module.amend_job_dict(job.data): changed = True if not changed: break @@ -247,9 +247,15 @@ class ModuleRegistry(object): component_data, component_type, eps, job_data, macro, name, xml_parent ) elif name in eps: - func = eps[name] - kwargs = self._filter_kwargs(func, job_data=job_data) - func(self, xml_parent, component_data, **kwargs) + try: + func = eps[name] + kwargs = self._filter_kwargs(func, job_data=job_data) + func(self, xml_parent, component_data, **kwargs) + except JenkinsJobsException as x: + raise x.with_context( + f"In {component_type} {name!r}", + pos=component.pos, + ) else: raise JenkinsJobsException( "Unknown entry point or macro '{0}' " @@ -280,7 +286,10 @@ class ModuleRegistry(object): try: element = expander.expand(b, expander_params) except JenkinsJobsException as x: - raise JenkinsJobsException(f"While expanding macro {name!r}: {x}") + raise x.with_context( + f"While expanding macro {name!r}", + pos=macro.pos, + ) # Pass component_data in as template data to this function # so that if the macro is invoked with arguments, # the arguments are interpolated into the real defn. diff --git a/jenkins_jobs/root_base.py b/jenkins_jobs/root_base.py index cbb6041a5..b6e6c58f9 100644 --- a/jenkins_jobs/root_base.py +++ b/jenkins_jobs/root_base.py @@ -11,14 +11,32 @@ # under the License. from collections import namedtuple -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import List from .constants import MAGIC_MANAGE_STRING -from .errors import JenkinsJobsException +from .errors import Context, JenkinsJobsException +from .loc_loader import LocDict, LocString +from .position import Pos from .formatter import enum_str_format_required_params, enum_str_format_param_defaults from .expander import Expander, expand_parameters from .defaults import Defaults -from .dimensions import DimensionsExpander +from .dimensions import enum_dimensions_params, is_point_included + + +@dataclass +class JobViewData: + """Expanded job or view data, with added source context. Fed into xml generator""" + + data: LocDict + context: List[Context] = field(default_factory=list) + + @property + def name(self): + return self.data["name"] + + def with_context(self, message, pos): + return JobViewData(self.data, [Context(message, pos), *self.context]) @dataclass @@ -30,6 +48,7 @@ class RootBase: _keep_descriptions: bool _id: str name: str + pos: Pos description: str defaults_name: str params: dict @@ -42,12 +61,19 @@ class RootBase: else: return self.name + @property + def title(self): + return str(self).capitalize() + def _format_description(self, params): if self.description is None: defaults = self._pick_defaults(self.defaults_name) description = defaults.params.get("description") else: - description = self.description + if type(self.description) is LocString: + description = str(self.description) + else: + description = self.description if description is None and self._keep_descriptions: return {} expanded_desc = self._expander.expand(description, params) @@ -60,8 +86,9 @@ class RootBase: if name == "global": return Defaults.empty() raise JenkinsJobsException( - f"Job template {self.name!r} wants defaults {self.defaults_name!r}" - " but it was never defined" + f"{self.title} wants defaults {self.defaults_name!r}" + " but it was never defined", + pos=name.pos, ) if name == "global": return defaults @@ -73,15 +100,21 @@ class RootBase: class NonTemplateRootMixin: def top_level_generate_items(self): - defaults = self._pick_defaults(self.defaults_name, merge_global=False) - description = self._format_description(params={}) - data = self._as_dict() - contents = self._expander.expand(data, self.params) - yield { - **defaults.contents, - **contents, - **description, - } + try: + defaults = self._pick_defaults(self.defaults_name, merge_global=False) + description = self._format_description(params={}) + raw_data = self._as_dict() + contents = self._expander.expand(raw_data, self.params) + data = LocDict.merge( + defaults.contents, + contents, + description, + pos=self.pos, + ) + context = [Context(f"In {self}", self.pos)] + yield JobViewData(data, context) + except JenkinsJobsException as x: + raise x.with_context(f"In {self}", pos=self.pos) def generate_items(self, defaults_name, params): # Do not produce jobs/views from under project - they are produced when @@ -91,62 +124,77 @@ class NonTemplateRootMixin: class TemplateRootMixin: def generate_items(self, defaults_name, params): - defaults = self._pick_defaults(defaults_name or self.defaults_name) - item_params = { - **defaults.params, - **self.params, - **params, - "template-name": self.name, - } - if self._id: - item_params["id"] = self._id - contents = { - **defaults.contents, - **self._as_dict(), - } - axes = list(enum_str_format_required_params(self.name)) - axes_defaults = dict(enum_str_format_param_defaults(self.name)) - dim_expander = DimensionsExpander(context=self.name) - for dim_params in dim_expander.enum_dimensions_params( - axes, item_params, axes_defaults - ): - instance_params = { - **item_params, - **dim_params, - } - expanded_params = expand_parameters( - self._expander, instance_params, template_name=self.name + try: + defaults = self._pick_defaults(defaults_name or self.defaults_name) + item_params = LocDict.merge( + defaults.params, + self.params, + params, + {"template-name": self.name}, ) - exclude_list = expanded_params.get("exclude") - if not dim_expander.is_point_included(exclude_list, expanded_params): - continue - description = self._format_description(expanded_params) - expanded_contents = self._expander.expand(contents, expanded_params) - yield { - **expanded_contents, - **description, - } + if self._id: + item_params["id"] = self._id + contents = LocDict.merge( + defaults.contents, + self._as_dict(), + ) + axes = list(enum_str_format_required_params(self.name, self.name.pos)) + axes_defaults = dict(enum_str_format_param_defaults(self.name)) + for dim_params in enum_dimensions_params(axes, item_params, axes_defaults): + instance_params = LocDict.merge( + item_params, + dim_params, + ) + expanded_params = expand_parameters(self._expander, instance_params) + if not is_point_included( + exclude_list=expanded_params.get("exclude"), + params=expanded_params, + key_pos=expanded_params.key_pos.get("exclude"), + ): + continue + description = self._format_description(expanded_params) + expanded_contents = self._expander.expand(contents, expanded_params) + data = LocDict.merge( + expanded_contents, + description, + pos=self.pos, + ) + context = [Context(f"In {self}", self.pos)] + yield JobViewData(data, context) + except JenkinsJobsException as x: + raise x.with_context(f"In {self}", pos=self.pos) class GroupBase: - Spec = namedtuple("Spec", "name params") + Spec = namedtuple("Spec", "name params pos") def __repr__(self): return f"<{self}>" @classmethod - def _spec_from_dict(cls, d, error_context): - if isinstance(d, str): - return cls.Spec(d, params={}) + def _specs_from_list(cls, spec_list=None): + if spec_list is None: + return [] + return [ + cls._spec_from_dict(item, spec_list.value_pos[idx]) + for idx, item in enumerate(spec_list) + ] + + @classmethod + def _spec_from_dict(cls, d, pos): + if isinstance(d, (str, LocString)): + return cls.Spec(d, params={}, pos=pos) if not isinstance(d, dict): raise JenkinsJobsException( - f"{error_context}: Job/view spec should name or dict," - f" but is {type(d)}. Missing indent?" + "Job/view spec should name or dict," + f" but is {type(d)} ({d!r}). Missing indent?", + pos=pos, ) if len(d) != 1: raise JenkinsJobsException( - f"{error_context}: Job/view dict should be single-item," - f" but have keys {list(d.keys())}. Missing indent?" + "Job/view dict should be single-item," + f" but have keys {list(d.keys())}. Missing indent?", + pos=d.pos, ) name, params = next(iter(d.items())) if params is None: @@ -154,40 +202,51 @@ class GroupBase: else: if not isinstance(params, dict): raise JenkinsJobsException( - f"{error_context}: Job/view {name} params type should be dict," - f" but is {type(params)} ({params})." + f"Job/view {name!r} params type should be dict," + f" but is {params!r}.", + pos=params.pos, ) - return cls.Spec(name, params) + return cls.Spec(name, params, pos) def _generate_items(self, root_dicts, spec_list, defaults_name, params): - for spec in spec_list: - item = self._pick_item(root_dicts, spec.name) - item_params = { - **params, - **self.params, - **self._my_params, - **spec.params, - } - yield from item.generate_items(defaults_name, item_params) + try: + for spec in spec_list: + item = self._pick_spec_item(root_dicts, spec) + item_params = LocDict.merge( + params, + self.params, + self._my_params, + spec.params, + ) + for job_data in item.generate_items(defaults_name, item_params): + yield ( + job_data.with_context("Defined here", spec.pos).with_context( + f"In {self}", self.pos + ) + ) + except JenkinsJobsException as x: + raise x.with_context(f"In {self}", self.pos) @property def _my_params(self): return {} - def _pick_item(self, root_dict_list, name): + def _pick_spec_item(self, root_dict_list, spec): for roots_dict in root_dict_list: try: - return roots_dict[name] + return roots_dict[spec.name] except KeyError: pass raise JenkinsJobsException( - f"{self}: Failed to find suitable job/view/template named '{name}'" + f"Failed to find suitable job/view/template named '{spec.name}'", + pos=spec.pos, ) @dataclass class Group(GroupBase): name: str + pos: Pos specs: list # list[Spec] params: dict diff --git a/jenkins_jobs/roots.py b/jenkins_jobs/roots.py index 29c464a02..5c5da9c57 100644 --- a/jenkins_jobs/roots.py +++ b/jenkins_jobs/roots.py @@ -13,7 +13,7 @@ import logging from collections import defaultdict -from .errors import JenkinsJobsException +from .errors import Context, JenkinsJobsException from .defaults import Defaults from .job import Job, JobTemplate, JobGroup from .view import View, ViewTemplate, ViewGroup @@ -57,7 +57,7 @@ class Roots: expanded_jobs += job.top_level_generate_items() for project in self.projects.values(): expanded_jobs += project.generate_jobs() - return self._remove_duplicates(expanded_jobs) + return self._remove_duplicates(expanded_jobs, "job") def generate_views(self): expanded_views = [] @@ -65,31 +65,44 @@ class Roots: expanded_views += view.top_level_generate_items() for project in self.projects.values(): expanded_views += project.generate_views() - return self._remove_duplicates(expanded_views) + return self._remove_duplicates(expanded_views, "view") - def assign(self, container, id, value, title): + def assign(self, container, id, value, element_type): if id in container: - self._handle_dups(f"Duplicate {title}: {id}") + self._handle_dups(element_type, id, value.pos, container[id].pos) container[id] = value - def _remove_duplicates(self, job_list): - seen = set() + def _remove_duplicates(self, job_or_view_list, element_type): + seen = {} unique_list = [] # Last definition wins. - for job in reversed(job_list): - name = job["name"] + for job_or_view in reversed(job_or_view_list): + name = job_or_view.name if name in seen: + origin = seen[name] self._handle_dups( - f"Duplicate definitions for job {name!r} specified", + element_type, + name, + job_or_view.data.pos, + origin.data.pos, + # Skip job context, leave only project context. + job_or_view.context[:-1], + origin.context[:-1], ) else: - unique_list.append(job) - seen.add(name) + unique_list.append(job_or_view) + seen[name] = job_or_view return unique_list[::-1] - def _handle_dups(self, message): + def _handle_dups( + self, element_type, id, pos, origin_pos, ctx=None, origin_ctx=None + ): + message = f"Duplicate {element_type}: {id!r}" if self._allow_duplicates: logger.warning(message) else: logger.error(message) - raise JenkinsJobsException(message) + ctx = [*(ctx or []), Context(message, pos), *(origin_ctx or [])] + raise JenkinsJobsException( + f"Previous {element_type} definition", origin_pos, ctx + ) diff --git a/jenkins_jobs/view.py b/jenkins_jobs/view.py index ea7d4223b..12a138cdf 100644 --- a/jenkins_jobs/view.py +++ b/jenkins_jobs/view.py @@ -12,6 +12,8 @@ from dataclasses import dataclass +from .errors import JenkinsJobsException +from .loc_loader import LocDict from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group from .defaults import split_contents_params, view_contents_keys @@ -21,14 +23,14 @@ class ViewBase(RootBase): view_type: str @classmethod - def from_dict(cls, config, roots, expander, data): + def from_dict(cls, config, roots, expander, data, pos): keep_descriptions = config.yamlparser["keep_descriptions"] - d = {**data} - name = d.pop("name") - id = d.pop("id", None) - description = d.pop("description", None) - defaults = d.pop("defaults", "global") - view_type = d.pop("view-type", "list") + d = data.copy() + name = d.pop_required_loc_string("name") + id = d.pop_loc_string("id", None) + description = d.pop_loc_string("description", None) + defaults = d.pop_loc_string("defaults", "global") + view_type = d.pop_loc_string("view-type", "list") contents, params = split_contents_params(d, view_contents_keys) return cls( roots.defaults, @@ -36,6 +38,7 @@ class ViewBase(RootBase): keep_descriptions, id, name, + pos, description, defaults, params, @@ -44,26 +47,34 @@ class ViewBase(RootBase): ) def _as_dict(self): - return { - "name": self.name, - "view-type": self.view_type, - **self.contents, - } + return LocDict.merge( + { + "name": self.name, + "view-type": self.view_type, + }, + self.contents, + ) class View(ViewBase, NonTemplateRootMixin): @classmethod - def add(cls, config, roots, expander, param_expander, data): - view = cls.from_dict(config, roots, expander, data) + def add(cls, config, roots, expander, param_expander, data, pos): + view = cls.from_dict(config, roots, expander, data, pos) roots.assign(roots.views, view.id, view, "view") + def __str__(self): + return f"view {self.name!r}" + class ViewTemplate(ViewBase, TemplateRootMixin): @classmethod - def add(cls, config, roots, expander, params_expander, data): - template = cls.from_dict(config, roots, params_expander, data) + def add(cls, config, roots, expander, params_expander, data, pos): + template = cls.from_dict(config, roots, params_expander, data, pos) roots.assign(roots.view_templates, template.id, template, "view template") + def __str__(self): + return f"view template {self.name!r}" + @dataclass class ViewGroup(Group): @@ -71,15 +82,16 @@ class ViewGroup(Group): _view_templates: dict @classmethod - def add(cls, config, roots, expander, params_expander, data): - d = {**data} - name = d.pop("name") - view_specs = [ - cls._spec_from_dict(item, error_context=f"View group {name}") - for item in d.pop("views") - ] + def add(cls, config, roots, expander, params_expander, data, pos): + d = data.copy() + name = d.pop_required_loc_string("name") + try: + view_specs = cls._specs_from_list(d.pop("views", None)) + except JenkinsJobsException as x: + raise x.with_context(f"In view {name!r}", pos=pos) group = cls( name, + pos, view_specs, d, roots.views, @@ -88,7 +100,7 @@ class ViewGroup(Group): roots.assign(roots.view_groups, group.name, group, "view group") def __str__(self): - return f"View group {self.name}" + return f"view group {self.name!r}" @property def _root_dicts(self): diff --git a/jenkins_jobs/xml_config.py b/jenkins_jobs/xml_config.py index ef5af761d..984b56844 100644 --- a/jenkins_jobs/xml_config.py +++ b/jenkins_jobs/xml_config.py @@ -21,7 +21,7 @@ import sys from xml.dom import minidom import xml.etree.ElementTree as XML -from jenkins_jobs import errors +from jenkins_jobs.errors import JenkinsJobsException __all__ = ["XmlJobGenerator", "XmlJob"] @@ -83,7 +83,10 @@ class XmlGenerator(object): def generateXML(self, data_list): xml_objs = [] for data in data_list: - xml_objs.append(self._getXMLForData(data)) + try: + xml_objs.append(self._getXMLForData(data.data)) + except JenkinsJobsException as x: + raise x.with_ctx_list(data.context) return xml_objs def _getXMLForData(self, data): @@ -104,7 +107,7 @@ class XmlGenerator(object): ep.name for ep in pkg_resources.iter_entry_points(group=self.entry_point_group) ] - raise errors.JenkinsJobsException( + raise JenkinsJobsException( "Unrecognized {}: {} (supported types are: {})".format( self.kind_attribute, kind, ", ".join(names) ) diff --git a/jenkins_jobs/yaml_objects.py b/jenkins_jobs/yaml_objects.py index 71148dd0e..f27642441 100644 --- a/jenkins_jobs/yaml_objects.py +++ b/jenkins_jobs/yaml_objects.py @@ -216,6 +216,7 @@ Examples: import abc import os.path import logging +import traceback import sys from pathlib import Path @@ -223,49 +224,50 @@ import jinja2 import jinja2.meta import yaml -from .errors import JenkinsJobsException +from .errors import Context, JenkinsJobsException +from .loc_loader import LocList +from .position import Pos from .formatter import CustomFormatter, enum_str_format_required_params -logger = logging.getLogger(__name__) - - if sys.version_info >= (3, 8): from functools import cached_property else: - from functools import lru_cache + from .cached_property import cached_property - # cached_property was introduced in python 3.8. - # Recipe from https://stackoverflow.com/a/19979379 - def cached_property(fn): - return property(lru_cache()(fn)) +logger = logging.getLogger(__name__) class BaseYamlObject(metaclass=abc.ABCMeta): @staticmethod def path_list_from_node(loader, node): if isinstance(node, yaml.ScalarNode): - return [loader.construct_yaml_str(node)] + return LocList( + [loader.construct_yaml_str(node)], + value_pos=[loader.pos_from_node(node)], + ) elif isinstance(node, yaml.SequenceNode): - return loader.construct_sequence(node) + return LocList( + loader.construct_sequence(node), + value_pos=[loader.pos_from_node(n) for n in node.value], + ) else: - raise yaml.constructor.ConstructorError( - None, - None, - f"expected either a sequence or scalar node, but found {node.id}", - node.start_mark, + raise JenkinsJobsException( + f"Expected either a sequence or scalar node, but found {node.id}", + pos=loader.pos_from_node(node), ) @classmethod def from_yaml(cls, loader, node): value = loader.construct_yaml_str(node) - return cls(loader.jjb_config, loader, value) + return cls(loader.jjb_config, loader, loader.pos_from_node(node), value) - def __init__(self, jjb_config, loader): + def __init__(self, jjb_config, loader, pos): self._search_path = jjb_config.yamlparser["include_path"] if loader.source_path: # Loaded from a file, find includes beside it too. self._search_path.append(os.path.dirname(loader.source_path)) self._loader = loader + self._pos = pos allow_empty = jjb_config.yamlparser["allow_empty_variables"] self._formatter = CustomFormatter(allow_empty) @@ -278,7 +280,7 @@ class BaseYamlObject(metaclass=abc.ABCMeta): """Expand object and substitute template parameters""" return self.expand(expander, params) - def _find_file(self, rel_path): + def _find_file(self, rel_path, pos): search_path = self._search_path if "." not in search_path: search_path.append(".") @@ -288,37 +290,60 @@ class BaseYamlObject(metaclass=abc.ABCMeta): if candidate.is_file(): logger.debug("Including file %r from path %r", str(rel_path), str(dir)) return candidate + dir_list_str = ",".join(str(d) for d in dir_list) raise JenkinsJobsException( - f"File {rel_path} does not exist on any of include directories:" - f" {','.join([str(d) for d in dir_list])}" + f"File {rel_path} does not exist in any of include directories: {dir_list_str}", + pos=pos, ) + def _expand_path_list(self, path_list, *args): + for idx, path in enumerate(path_list): + yield self._expand_path(path, path_list.value_pos[idx], *args) + + def _subst_path_list(self, path_list, *args): + for idx, path in enumerate(path_list): + yield self._subst_path(path, path_list.value_pos[idx], *args) + class J2BaseYamlObject(BaseYamlObject): - def __init__(self, jjb_config, loader): - super().__init__(jjb_config, loader) + def __init__(self, jjb_config, loader, pos): + super().__init__(jjb_config, loader, pos) self._jinja2_env = jinja2.Environment( loader=jinja2.FileSystemLoader(self._search_path), undefined=jinja2.StrictUndefined, ) - @staticmethod - def _render_template(template_text, template, params): + def _render_template(self, pos, template_text, template, params): try: return template.render(params) except jinja2.UndefinedError as x: + # Jinja2 adds fake traceback entry with template line number. + tb = traceback.extract_tb(x.__traceback__) + line_ofs = tb[-1].lineno - 1 # traceback lineno starts with 1. + lines = template_text.splitlines() + start_ofs = pos.body.index(lines[0]) + # Examples for pre_pad: '!j2: \n', '!j2: '. + pre_pad = pos.body[:start_ofs] + # Shift position to reflect template position inside yaml file: + if "\n" in pre_pad: + pos = pos.with_offset(line_ofs=1) + column_ofs = 0 + else: + column_ofs = start_ofs + # Move position to error inside template: + pos = pos.with_offset(line_ofs, column_ofs) + pos = pos.with_contents_start() if len(template_text) > 40: text = template_text[:40] + "..." else: text = template_text - raise JenkinsJobsException( - f"While formatting jinja2 template {text!r}: {x}" - ) + context = Context(f"While formatting jinja2 template {text!r}", self._pos) + raise JenkinsJobsException(str(x), pos=pos, ctx=[context]) class J2Template(J2BaseYamlObject): - def __init__(self, jjb_config, loader, template_text): - super().__init__(jjb_config, loader) + def __init__(self, jjb_config, loader, pos, template_text): + super().__init__(jjb_config, loader, pos) self._template_text = template_text self._template = self._jinja2_env.from_string(template_text) @@ -328,7 +353,9 @@ class J2Template(J2BaseYamlObject): return jinja2.meta.find_undeclared_variables(ast) def _render(self, params): - return self._render_template(self._template_text, self._template, params) + return self._render_template( + self._pos, self._template_text, self._template, params + ) class J2String(J2Template): @@ -343,8 +370,11 @@ class J2Yaml(J2Template): def expand(self, expander, params): text = self._render(params) - data = self._loader.load(text) - return expander.expand(data, params) + data = self._loader.load(text, source_path="") + try: + return expander.expand(data, params) + except JenkinsJobsException as x: + raise x.with_context("In expanded !j2-yaml:", self._pos) class IncludeJinja2(J2BaseYamlObject): @@ -353,10 +383,10 @@ class IncludeJinja2(J2BaseYamlObject): @classmethod def from_yaml(cls, loader, node): path_list = cls.path_list_from_node(loader, node) - return cls(loader.jjb_config, loader, path_list) + return cls(loader.jjb_config, loader, loader.pos_from_node(node), path_list) - def __init__(self, jjb_config, loader, path_list): - super().__init__(jjb_config, loader) + def __init__(self, jjb_config, loader, pos, path_list): + super().__init__(jjb_config, loader, pos) self._path_list = path_list @property @@ -364,91 +394,98 @@ class IncludeJinja2(J2BaseYamlObject): return [] def expand(self, expander, params): - return "\n".join( - self._expand_path(expander, params, path) for path in self._path_list - ) + return "\n".join(self._expand_path_list(self._path_list, expander, params)) - def _expand_path(self, expander, params, path_template): + def _expand_path(self, path_template, pos, expander, params): rel_path = self._formatter.format(path_template, **params) - full_path = self._find_file(rel_path) + full_path = self._find_file(rel_path, pos) template_text = full_path.read_text() template = self._jinja2_env.from_string(template_text) - return self._render_template(template_text, template, params) + pos = Pos.from_file(full_path, template_text) + try: + return self._render_template(pos, template_text, template, params) + except JenkinsJobsException as x: + raise x.with_context(f"In included file {str(full_path)!r}", pos=self._pos) class IncludeBaseObject(BaseYamlObject): @classmethod def from_yaml(cls, loader, node): path_list = cls.path_list_from_node(loader, node) - return cls(loader.jjb_config, loader, path_list) + return cls(loader.jjb_config, loader, loader.pos_from_node(node), path_list) - def __init__(self, jjb_config, loader, path_list): - super().__init__(jjb_config, loader) + def __init__(self, jjb_config, loader, pos, path_list): + super().__init__(jjb_config, loader, pos) self._path_list = path_list @property def required_params(self): - for path in self._path_list: - yield from enum_str_format_required_params(path) + for idx, path in enumerate(self._path_list): + yield from enum_str_format_required_params( + path, pos=self._path_list.value_pos[idx] + ) class YamlInclude(IncludeBaseObject): yaml_tag = "!include:" def expand(self, expander, params): - yaml_list = [ - self._expand_path(expander, params, path) for path in self._path_list - ] + yaml_list = list(self._expand_path_list(self._path_list, expander, params)) if len(yaml_list) == 1: return yaml_list[0] else: return "\n".join(yaml_list) - def _expand_path(self, expander, params, path_template): + def _expand_path(self, path_template, pos, expander, params): rel_path = self._formatter.format(path_template, **params) - full_path = self._find_file(rel_path) - text = full_path.read_text() - data = self._loader.load(text) - return expander.expand(data, params) + full_path = self._find_file(rel_path, pos) + data = self._loader.load_path(full_path) + try: + return expander.expand(data, params) + except JenkinsJobsException as x: + raise x.with_context(f"In included file {str(full_path)!r}", pos=self._pos) class IncludeRawBase(IncludeBaseObject): def expand(self, expander, params): - return "\n".join(self._expand_path(path, params) for path in self._path_list) + return "\n".join(self._expand_path_list(self._path_list, params)) def subst(self, expander, params): - return "\n".join(self._subst_path(path, params) for path in self._path_list) + return "\n".join(self._subst_path_list(self._path_list, params)) class IncludeRaw(IncludeRawBase): yaml_tag = "!include-raw:" - def _expand_path(self, rel_path_template, params): + def _expand_path(self, rel_path_template, pos, params): rel_path = self._formatter.format(rel_path_template, **params) - full_path = self._find_file(rel_path) + full_path = self._find_file(rel_path, pos) return full_path.read_text() - def _subst_path(self, rel_path_template, params): + def _subst_path(self, rel_path_template, pos, params): rel_path = self._formatter.format(rel_path_template, **params) - full_path = self._find_file(rel_path) + full_path = self._find_file(rel_path, pos) template = full_path.read_text() - return self._formatter.format(template, **params) + try: + return self._formatter.format(template, **params) + except JenkinsJobsException as x: + raise x.with_context(f"In included file {str(full_path)!r}", pos=self._pos) class IncludeRawEscape(IncludeRawBase): yaml_tag = "!include-raw-escape:" - def _expand_path(self, rel_path_template, params): + def _expand_path(self, rel_path_template, pos, params): rel_path = self._formatter.format(rel_path_template, **params) - full_path = self._find_file(rel_path) + full_path = self._find_file(rel_path, pos) text = full_path.read_text() # Backward compatibility: # if used inside job or macro without parameters, curly braces are duplicated. return text.replace("{", "{{").replace("}", "}}") - def _subst_path(self, rel_path_template, params): + def _subst_path(self, rel_path_template, pos, params): rel_path = self._formatter.format(rel_path_template, **params) - full_path = self._find_file(rel_path) + full_path = self._find_file(rel_path, pos) return full_path.read_text() @@ -459,12 +496,10 @@ class YamlListJoin: def from_yaml(cls, loader, node): value = loader.construct_sequence(node, deep=True) if len(value) != 2: - raise yaml.constructor.ConstructorError( - None, - None, + raise JenkinsJobsException( "Join value should contain 2 elements: delimiter and string list," f" but contains {len(value)} elements: {value!r}", - node.start_mark, + pos=loader.pos_from_node(node), ) delimiter, seq = value return delimiter.join(seq) diff --git a/tests/duplicates/job_fixtures/exception_duplicates001.error b/tests/duplicates/job_fixtures/exception_duplicates001.error new file mode 100644 index 000000000..7d8ad3ac5 --- /dev/null +++ b/tests/duplicates/job_fixtures/exception_duplicates001.error @@ -0,0 +1,6 @@ +exception_duplicates001.yaml:7:3: Duplicate job: 'duplicate001' + - job: + ^ +exception_duplicates001.yaml:1:3: Previous job definition + - job: + ^ diff --git a/tests/duplicates/job_fixtures/exception_duplicates002.error b/tests/duplicates/job_fixtures/exception_duplicates002.error new file mode 100644 index 000000000..5f951e2ff --- /dev/null +++ b/tests/duplicates/job_fixtures/exception_duplicates002.error @@ -0,0 +1,12 @@ +exception_duplicates002.yaml:14:3: Duplicate job: 'duplicates002_1.1' + - job: + ^ +exception_duplicates002.yaml:1:3: In project 'duplicates' + - project: + ^ +exception_duplicates002.yaml:6:11: Defined here + - 'duplicates002_{version}' + ^ +exception_duplicates002.yaml:8:3: Previous job definition + - job-template: + ^ diff --git a/tests/duplicates/job_fixtures/exception_job_group001.error b/tests/duplicates/job_fixtures/exception_job_group001.error new file mode 100644 index 000000000..63eb05960 --- /dev/null +++ b/tests/duplicates/job_fixtures/exception_job_group001.error @@ -0,0 +1,6 @@ +exception_job_group001.yaml:14:3: Duplicate job group: 'group-1' + - job-group: + ^ +exception_job_group001.yaml:11:3: Previous job group definition + - job-group: + ^ diff --git a/tests/duplicates/job_fixtures/exception_macros001.error b/tests/duplicates/job_fixtures/exception_macros001.error new file mode 100644 index 000000000..f4e60261a --- /dev/null +++ b/tests/duplicates/job_fixtures/exception_macros001.error @@ -0,0 +1,6 @@ +exception_macros001.yaml:9:3: Duplicate macro: 'project-scm' + - scm: + ^ +exception_macros001.yaml:1:3: Previous macro definition + - scm: + ^ diff --git a/tests/duplicates/job_fixtures/exception_projects001.error b/tests/duplicates/job_fixtures/exception_projects001.error new file mode 100644 index 000000000..1de63ffff --- /dev/null +++ b/tests/duplicates/job_fixtures/exception_projects001.error @@ -0,0 +1,18 @@ +exception_projects001.yaml:1:3: In project 'duplicates' + - project: + ^ +exception_projects001.yaml:6:11: Defined here + - 'duplicates002_1.1' + ^ +exception_projects001.yaml:9:3: Duplicate job: 'duplicates002_1.1' + - job-template: + ^ +exception_projects001.yaml:1:3: In project 'duplicates' + - project: + ^ +exception_projects001.yaml:7:11: Defined here + - 'duplicates002_1.1' + ^ +exception_projects001.yaml:9:3: Previous job definition + - job-template: + ^ diff --git a/tests/duplicates/job_fixtures/exception_projects002.error b/tests/duplicates/job_fixtures/exception_projects002.error new file mode 100644 index 000000000..314653d52 --- /dev/null +++ b/tests/duplicates/job_fixtures/exception_projects002.error @@ -0,0 +1,30 @@ +exception_projects002.yaml:1:3: In project 'duplicate-groups' + - project: + ^ +exception_projects002.yaml:4:9: Defined here + - 'group-001' + ^ +exception_projects002.yaml:7:3: In job group 'group-001' + - job-group: + ^ +exception_projects002.yaml:10:9: Defined here + - dummy-job + ^ +exception_projects002.yaml:12:3: Duplicate job: 'dummy-job' + - job-template: + ^ +exception_projects002.yaml:1:3: In project 'duplicate-groups' + - project: + ^ +exception_projects002.yaml:5:9: Defined here + - 'group-001' + ^ +exception_projects002.yaml:7:3: In job group 'group-001' + - job-group: + ^ +exception_projects002.yaml:10:9: Defined here + - dummy-job + ^ +exception_projects002.yaml:12:3: Previous job definition + - job-template: + ^ diff --git a/tests/duplicates/job_fixtures/exception_projects003.error b/tests/duplicates/job_fixtures/exception_projects003.error new file mode 100644 index 000000000..48a3832a2 --- /dev/null +++ b/tests/duplicates/job_fixtures/exception_projects003.error @@ -0,0 +1,18 @@ +exception_projects003.yaml:1:3: In project 'duplicate-templates' + - project: + ^ +exception_projects003.yaml:4:11: Defined here + - '{name}-001' + ^ +exception_projects003.yaml:7:3: Duplicate job: 'duplicate-templates-001' + - job-template: + ^ +exception_projects003.yaml:1:3: In project 'duplicate-templates' + - project: + ^ +exception_projects003.yaml:5:11: Defined here + - '{name}-001' + ^ +exception_projects003.yaml:7:3: Previous job definition + - job-template: + ^ diff --git a/tests/duplicates/job_fixtures/exception_templates001.error b/tests/duplicates/job_fixtures/exception_templates001.error new file mode 100644 index 000000000..9de084ec5 --- /dev/null +++ b/tests/duplicates/job_fixtures/exception_templates001.error @@ -0,0 +1,6 @@ +exception_templates001.yaml:12:3: Duplicate job template: '{name}-001' + - job-template: + ^ +exception_templates001.yaml:6:3: Previous job template definition + - job-template: + ^ diff --git a/tests/duplicates/test_job_duplicates.py b/tests/duplicates/test_job_duplicates.py index db2e03b97..68b4b3928 100644 --- a/tests/duplicates/test_job_duplicates.py +++ b/tests/duplicates/test_job_duplicates.py @@ -33,10 +33,13 @@ def scenario(request): return request.param -def test_yaml_snippet(scenario, check_job): - if scenario.in_path.name.startswith("exception_"): +def test_yaml_snippet(scenario, expected_error, check_job): + if scenario.error_path.exists(): with pytest.raises(JenkinsJobsException) as excinfo: check_job() - assert str(excinfo.value).startswith("Duplicate ") + error = "\n".join(excinfo.value.lines) + print() + print(error) + assert error.replace(str(fixtures_dir) + "/", "") == expected_error else: check_job() diff --git a/tests/duplicates/test_view_duplicates.py b/tests/duplicates/test_view_duplicates.py index 280f464ad..b5d697e49 100644 --- a/tests/duplicates/test_view_duplicates.py +++ b/tests/duplicates/test_view_duplicates.py @@ -18,10 +18,13 @@ def scenario(request): return request.param -def test_yaml_snippet(scenario, check_view): - if scenario.in_path.name.startswith("exception_"): +def test_yaml_snippet(scenario, expected_error, check_view): + if scenario.error_path.exists(): with pytest.raises(JenkinsJobsException) as excinfo: check_view() - assert str(excinfo.value).startswith("Duplicate ") + error = "\n".join(excinfo.value.lines) + print() + print(error) + assert error.replace(str(fixtures_dir) + "/", "") == expected_error else: check_view() diff --git a/tests/duplicates/view_fixtures/exception_duplicate_view.error b/tests/duplicates/view_fixtures/exception_duplicate_view.error new file mode 100644 index 000000000..d52ea382e --- /dev/null +++ b/tests/duplicates/view_fixtures/exception_duplicate_view.error @@ -0,0 +1,6 @@ +exception_duplicate_view.yaml:5:3: Duplicate view: 'duplicate-view' + - view: + ^ +exception_duplicate_view.yaml:1:3: Previous view definition + - view: + ^ diff --git a/tests/duplicates/view_fixtures/exception_duplicate_view_template.error b/tests/duplicates/view_fixtures/exception_duplicate_view_template.error new file mode 100644 index 000000000..c377e6a75 --- /dev/null +++ b/tests/duplicates/view_fixtures/exception_duplicate_view_template.error @@ -0,0 +1,6 @@ +exception_duplicate_view_template.yaml:5:3: Duplicate view template: 'duplicate-view-template' + - view-template: + ^ +exception_duplicate_view_template.yaml:1:3: Previous view template definition + - view-template: + ^ diff --git a/tests/duplicates/view_fixtures/exception_duplicate_views_in_project.error b/tests/duplicates/view_fixtures/exception_duplicate_views_in_project.error new file mode 100644 index 000000000..4dd10917d --- /dev/null +++ b/tests/duplicates/view_fixtures/exception_duplicate_views_in_project.error @@ -0,0 +1,18 @@ +exception_duplicate_views_in_project.yaml:5:3: In project 'sample-project' + - project: + ^ +exception_duplicate_views_in_project.yaml:8:9: Defined here + - sample-template + ^ +exception_duplicate_views_in_project.yaml:1:3: Duplicate view: 'sample-template' + - view-template: + ^ +exception_duplicate_views_in_project.yaml:5:3: In project 'sample-project' + - project: + ^ +exception_duplicate_views_in_project.yaml:9:9: Defined here + - sample-template + ^ +exception_duplicate_views_in_project.yaml:1:3: Previous view definition + - view-template: + ^ diff --git a/tests/formatter/test_formatter.py b/tests/formatter/test_formatter.py index 4e766f9e1..d9ab02f27 100644 --- a/tests/formatter/test_formatter.py +++ b/tests/formatter/test_formatter.py @@ -1,6 +1,7 @@ import pytest from jinja2 import StrictUndefined +from jenkins_jobs.errors import JenkinsJobsException from jenkins_jobs.formatter import ( CustomFormatter, enum_str_format_required_params, @@ -144,7 +145,7 @@ def test_format(format, vars, used_vars, expected_defaults, expected_result): def test_used_params( format, vars, expected_used_vars, expected_defaults, expected_result ): - used_vars = set(enum_str_format_required_params(format)) + used_vars = set(enum_str_format_required_params(format, pos=None)) assert used_vars == set(expected_used_vars) @@ -193,7 +194,7 @@ positional_cases = [ @pytest.mark.parametrize("format", positional_cases) def test_positional_args(format): formatter = CustomFormatter(allow_empty=False) - with pytest.raises(RuntimeError) as excinfo: + with pytest.raises(JenkinsJobsException) as excinfo: list(formatter.enum_required_params(format)) message = f"Positional format arguments are not supported: {format!r}" assert str(excinfo.value) == message diff --git a/tests/formatter/test_jinja2.py b/tests/formatter/test_jinja2.py index 0a9d633b4..35eee0045 100644 --- a/tests/formatter/test_jinja2.py +++ b/tests/formatter/test_jinja2.py @@ -17,5 +17,5 @@ cases = [ def test_jinja2_required_params(format, expected_used_params): config = JJBConfig() loader = Mock(source_path=None) - template = J2String(config, loader, format) + template = J2String(config, loader, pos=None, template_text=format) assert template.required_params == set(expected_used_params) diff --git a/tests/loader/loc_fixtures/sample_01.yaml b/tests/loader/loc_fixtures/sample_01.yaml new file mode 100644 index 000000000..dae9f55cf --- /dev/null +++ b/tests/loader/loc_fixtures/sample_01.yaml @@ -0,0 +1,29 @@ +- job_template: + name: sample-job-1 + builders: &job_builders + - shell: | + #!/usr/bin/env bash -xe + echo this is sample bash script + and this is it's third line + - sample_macro: ¯o_params + param_1: value_1 + param_2: value_2 + +- job_template: + name: sample-job-2 + builders: *job_builders + +- job_template: + name: sample-job-3 + builders: + - sample_macro: + <<: *macro_params + param_3: value_3 + +- project: + name: sample-project + param_1: sample_value + jobs: + - sample-job-1 + - sample-job-2 + - sample-job-3 diff --git a/tests/loader/test_loader.py b/tests/loader/test_loader.py index ef086e54f..33b1ea018 100644 --- a/tests/loader/test_loader.py +++ b/tests/loader/test_loader.py @@ -62,7 +62,7 @@ def test_include(scenario, jjb_config, expected_output): roots = Roots(jjb_config) load_files(jjb_config, roots, [scenario.in_path]) - job_data_list = roots.generate_jobs() + job_data_list = [j.data for j in roots.generate_jobs()] pretty_json = json.dumps(job_data_list, indent=4) print(pretty_json) assert pretty_json == expected_output.strip() @@ -198,4 +198,4 @@ def test_retain_anchors_enabled_j2_yaml(): registry = ModuleRegistry(config, None) registry.set_macros(roots.macros) jobs = roots.generate_jobs() - assert "docker run ubuntu:latest" == jobs[0]["builders"][0]["shell"] + assert "docker run ubuntu:latest" == jobs[0].data["builders"][0]["shell"] diff --git a/tests/loader/test_locations.py b/tests/loader/test_locations.py new file mode 100644 index 000000000..b75fc4cf7 --- /dev/null +++ b/tests/loader/test_locations.py @@ -0,0 +1,72 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pathlib import Path + +from jenkins_jobs.loc_loader import LocLoader + +fixtures_dir = Path(__file__).parent / "loc_fixtures" + + +def test_location(): + print() + path = fixtures_dir / "sample_01.yaml" + loader = LocLoader(path.read_text(), str(path)) + data = loader.get_single_data() + + b = data[0]["job_template"]["builders"][0] + print(type(b), b.pos) + print(b) + print(b.pos.snippet) + + b = data[0]["job_template"]["builders"][1]["sample_macro"] + print(type(b), b.pos) + print(b) + print("items", b.value_pos) + for key, pos in b.value_pos.items(): + print("item:", key, pos) + print(pos.snippet) + + b = data[0]["job_template"]["builders"] + print(type(b), b.pos) + print(b) + print("items", b.value_pos) + for idx, pos in enumerate(b.value_pos): + print("item:", idx, pos) + + print("--- sample-job-2 builders -------------------------------------------") + b = data[1]["job_template"]["builders"] + print(type(b), b.pos) + print(b) + print("items", b.value_pos) + for idx, pos in enumerate(b.value_pos): + print("item:", idx, pos) + print(pos.snippet) + + print("--- sample-job-2 sample_macro ---------------------------------------") + sample_macro = data[1]["job_template"]["builders"][1]["sample_macro"] + param_2_pos = sample_macro.value_pos["param_2"] + print(param_2_pos) + print(param_2_pos.snippet) + assert param_2_pos.line == 9 + assert param_2_pos.column == 19 + assert param_2_pos.snippet.splitlines()[0].strip() == "param_2: value_2" + + print("--- third template -------------------------------------------------") + b = data[2]["job_template"]["builders"][0]["sample_macro"] + print(type(b), b.pos) + print(b) + print("items", b.value_pos) + for key, pos in b.key_pos.items(): + print("keys for item:", key, pos) + for key, pos in b.value_pos.items(): + print("values for item:", key, pos) diff --git a/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type001.error b/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type001.error index 4febda221..93ec65dd5 100644 --- a/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type001.error +++ b/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type001.error @@ -1 +1,6 @@ -'['test']' is an invalid value for attribute name.trigger-build-on-pr-comment +scm_github_comment_plugin_invalid_type001.yaml:12:19: For attribute 'trigger-build-on-pr-comment' + - trigger-build-on-pr-comment: + ^ +scm_github_comment_plugin_invalid_type001.yaml:13:21: '['test']' is an invalid value for attribute name.trigger-build-on-pr-comment + - test + ^ diff --git a/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type002.error b/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type002.error index 9fe248031..8c6f16afd 100644 --- a/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type002.error +++ b/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type002.error @@ -1 +1,6 @@ -'['test']' is an invalid value for attribute name.trigger-build-on-pr-review +scm_github_comment_plugin_invalid_type002.yaml:14:19: For attribute 'trigger-build-on-pr-review' + - trigger-build-on-pr-review: + ^ +scm_github_comment_plugin_invalid_type002.yaml:15:21: '['test']' is an invalid value for attribute name.trigger-build-on-pr-review + - test + ^ diff --git a/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type003.error b/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type003.error index 8d2f768be..2367521b6 100644 --- a/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type003.error +++ b/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type003.error @@ -1 +1,6 @@ -'['test']' is an invalid value for attribute name.trigger-build-on-pr-update +scm_github_comment_plugin_invalid_type003.yaml:16:19: For attribute 'trigger-build-on-pr-update' + - trigger-build-on-pr-update: + ^ +scm_github_comment_plugin_invalid_type003.yaml:17:21: '['test']' is an invalid value for attribute name.trigger-build-on-pr-update + - test + ^ diff --git a/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type004.error b/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type004.error index c53b6fbf6..382d281c9 100644 --- a/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type004.error +++ b/tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type004.error @@ -1 +1,6 @@ -'true' is an invalid value for attribute name.trigger-build-on-pr-update +scm_github_comment_plugin_invalid_type004.yaml:16:19: For attribute 'trigger-build-on-pr-update' + - trigger-build-on-pr-update: "true" + ^ +scm_github_comment_plugin_invalid_type004.yaml:16:47: 'true' is an invalid value for attribute name.trigger-build-on-pr-update + - trigger-build-on-pr-update: "true" + ^ diff --git a/tests/multibranch/error_fixtures/scm_github_comment_plugin_missing_comment.error b/tests/multibranch/error_fixtures/scm_github_comment_plugin_missing_comment.error index 1ca197254..b6ad68b3d 100644 --- a/tests/multibranch/error_fixtures/scm_github_comment_plugin_missing_comment.error +++ b/tests/multibranch/error_fixtures/scm_github_comment_plugin_missing_comment.error @@ -1 +1,3 @@ -Missing trigger-build-on-pr-comment[comment] from an instance of 'name' +scm_github_comment_plugin_missing_comment.yaml:12:19: Missing trigger-build-on-pr-comment[comment] from an instance of 'name' + - trigger-build-on-pr-comment: + ^ diff --git a/tests/multibranch/test_errors.py b/tests/multibranch/test_errors.py index 522175235..aa4027b7c 100644 --- a/tests/multibranch/test_errors.py +++ b/tests/multibranch/test_errors.py @@ -17,6 +17,7 @@ from pathlib import Path import pytest +from jenkins_jobs.errors import JenkinsJobsException from jenkins_jobs.modules import project_multibranch from tests.enum_scenarios import scenario_list @@ -32,6 +33,9 @@ def scenario(request): def test_error(check_generator, expected_error): - with pytest.raises(Exception) as excinfo: + with pytest.raises(JenkinsJobsException) as excinfo: check_generator(project_multibranch.WorkflowMultiBranch) - assert str(excinfo.value) == expected_error + error = "\n".join(excinfo.value.lines) + print() + print(error) + assert error.replace(str(fixtures_dir) + "/", "") == expected_error diff --git a/tests/xml_config/test_xml_config.py b/tests/xml_config/test_xml_config.py index 07fb9afab..aeb32cb0f 100644 --- a/tests/xml_config/test_xml_config.py +++ b/tests/xml_config/test_xml_config.py @@ -77,10 +77,7 @@ def test_template_params(parser, registry): with pytest.raises(Exception) as excinfo: generator.generateXML(jobs) - message = ( - "While expanding macro 'default-git-scm':" - " While formatting string '{branches}': Missing parameter: 'branches'" - ) + message = "While formatting string '{branches}': Missing parameter: 'branches'" assert str(excinfo.value) == message @@ -91,10 +88,7 @@ def test_missing_j2_param(parser, registry): with pytest.raises(Exception) as excinfo: generator.generateXML(jobs) - message = ( - "While expanding macro 'default-git-scm':" - " While formatting jinja2 template '{{ branches }}': 'branches' is undefined" - ) + message = "'branches' is undefined" assert str(excinfo.value) == message @@ -105,9 +99,5 @@ def test_missing_include_j2_param(parser, registry): with pytest.raises(Exception) as excinfo: generator.generateXML(jobs) - message = ( - "While expanding macro 'a-builder':" - " While formatting jinja2 template 'echo \"Parameter branch={{ branches }} is...':" - " 'branches' is undefined" - ) + message = "'branches' is undefined" assert str(excinfo.value) == message diff --git a/tests/yamlparser/error_fixtures/failure_formatting_indent.error b/tests/yamlparser/error_fixtures/failure_formatting_indent.error index 3af3db0b7..14a72d3f3 100644 --- a/tests/yamlparser/error_fixtures/failure_formatting_indent.error +++ b/tests/yamlparser/error_fixtures/failure_formatting_indent.error @@ -1 +1,6 @@ -Project missing_params_for_params: Job/view dict should be single-item, but have keys ['template-requiring-param-{os}', 'os']. Missing indent? +failure_formatting_indent.yaml:5:3: In project 'missing_params_for_params' + - project: + ^ +failure_formatting_indent.yaml:13:9: Job/view dict should be single-item, but have keys ['template-requiring-param-{os}', 'os']. Missing indent? + - 'template-requiring-param-{os}': + ^ diff --git a/tests/yamlparser/error_fixtures/failure_formatting_params.error b/tests/yamlparser/error_fixtures/failure_formatting_params.error index 5461ac39d..e570e5515 100644 --- a/tests/yamlparser/error_fixtures/failure_formatting_params.error +++ b/tests/yamlparser/error_fixtures/failure_formatting_params.error @@ -1 +1,12 @@ -While expanding 'flavor', used by , used by template 'template-requiring-param-{os}': While formatting string 'xenial-{bdate}': 'bdate' is undefined +failure_formatting_params.yaml:5:3: In project 'missing_params_for_params' + - project: + ^ +failure_formatting_params.yaml:16:3: In job template 'template-requiring-param-{os}' + - job-template: + ^ +failure_formatting_params.yaml:9:5: While expanding parameter 'flavor' + flavor: + ^ +failure_formatting_params.yaml:11:9: While formatting string 'xenial-{bdate}': 'bdate' is undefined + - xenial-{bdate} + ^ diff --git a/tests/yamlparser/error_fixtures/failure_formatting_template.error b/tests/yamlparser/error_fixtures/failure_formatting_template.error index 8f5f747c0..e2a8dddaf 100644 --- a/tests/yamlparser/error_fixtures/failure_formatting_template.error +++ b/tests/yamlparser/error_fixtures/failure_formatting_template.error @@ -1 +1,9 @@ -While formatting string 'template-requiring-param-{os}': Missing parameter: 'os' +failure_formatting_template.yaml:1:3: In project 'missing_params_for_template' + - project: + ^ +failure_formatting_template.yaml:6:3: In job template 'template-requiring-param-{os}' + - job-template: + ^ +failure_formatting_template.yaml:7:12: While formatting string 'template-requiring-param-{os}': Missing parameter: 'os' + name: 'template-requiring-param-{os}' + ^ diff --git a/tests/yamlparser/error_fixtures/format_positional_argument_in_job_name.error b/tests/yamlparser/error_fixtures/format_positional_argument_in_job_name.error new file mode 100644 index 000000000..43ec8da7a --- /dev/null +++ b/tests/yamlparser/error_fixtures/format_positional_argument_in_job_name.error @@ -0,0 +1,9 @@ +format_positional_argument_in_job_name.yaml:6:3: In project 'sample-project' + - project: + ^ +format_positional_argument_in_job_name.yaml:1:3: In job template 'sample-job-{0}' + - job-template: + ^ +format_positional_argument_in_job_name.yaml:2:11: Positional format arguments are not supported: 'sample-job-{0}' + name: sample-job-{0} + ^ diff --git a/tests/yamlparser/error_fixtures/format_positional_argument_in_job_name.yaml b/tests/yamlparser/error_fixtures/format_positional_argument_in_job_name.yaml new file mode 100644 index 000000000..0dc905eb6 --- /dev/null +++ b/tests/yamlparser/error_fixtures/format_positional_argument_in_job_name.yaml @@ -0,0 +1,9 @@ +- job-template: + name: sample-job-{0} + builders: + - shell: echo ok + +- project: + name: sample-project + jobs: + - sample-job-{0} diff --git a/tests/yamlparser/error_fixtures/format_positional_argument_in_param.error b/tests/yamlparser/error_fixtures/format_positional_argument_in_param.error new file mode 100644 index 000000000..a3aa5aa69 --- /dev/null +++ b/tests/yamlparser/error_fixtures/format_positional_argument_in_param.error @@ -0,0 +1,9 @@ +format_positional_argument_in_param.yaml:6:3: In project 'sample-project' + - project: + ^ +format_positional_argument_in_param.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +format_positional_argument_in_param.yaml:8:12: Positional format arguments are not supported: 'positional-format-{0}' + param: 'positional-format-{0}' + ^ diff --git a/tests/yamlparser/error_fixtures/format_positional_argument_in_param.yaml b/tests/yamlparser/error_fixtures/format_positional_argument_in_param.yaml new file mode 100644 index 000000000..60bd6d306 --- /dev/null +++ b/tests/yamlparser/error_fixtures/format_positional_argument_in_param.yaml @@ -0,0 +1,10 @@ +- job-template: + name: sample-job + builders: + - shell: echo {param} + +- project: + name: sample-project + param: 'positional-format-{0}' + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/include_jinja2_missing_path.error b/tests/yamlparser/error_fixtures/include_jinja2_missing_path.error new file mode 100644 index 000000000..64d366c72 --- /dev/null +++ b/tests/yamlparser/error_fixtures/include_jinja2_missing_path.error @@ -0,0 +1,9 @@ +include_jinja2_missing_path.yaml:6:3: In project 'sample-project' + - project: + ^ +include_jinja2_missing_path.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +include_jinja2_missing_path.yaml:4:16: File missing-path.inc does not exist in any of include directories: .,fixtures-dir + - shell: !include-jinja2: missing-path.inc + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct.yaml b/tests/yamlparser/error_fixtures/include_jinja2_missing_path.yaml similarity index 67% rename from tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct.yaml rename to tests/yamlparser/error_fixtures/include_jinja2_missing_path.yaml index 253a701d0..6f666382c 100644 --- a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct.yaml +++ b/tests/yamlparser/error_fixtures/include_jinja2_missing_path.yaml @@ -1,10 +1,9 @@ +- job-template: + name: sample-job + builders: + - shell: !include-jinja2: missing-path.inc + - project: name: sample-project jobs: - sample-job - -- job-template: - name: sample-job - builders: - - shell: !j2: | - echo {{ missing_param }} diff --git a/tests/yamlparser/error_fixtures/include_missing_path.error b/tests/yamlparser/error_fixtures/include_missing_path.error new file mode 100644 index 000000000..80a830765 --- /dev/null +++ b/tests/yamlparser/error_fixtures/include_missing_path.error @@ -0,0 +1,9 @@ +include_missing_path.yaml:7:3: In project 'sample-project' + - project: + ^ +include_missing_path.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +include_missing_path.yaml:5:11: File missing-file.sh does not exist in any of include directories: .,fixtures-dir + !include: missing-file.sh + ^ diff --git a/tests/yamlparser/error_fixtures/include_missing_path.yaml b/tests/yamlparser/error_fixtures/include_missing_path.yaml new file mode 100644 index 000000000..d6685b6b7 --- /dev/null +++ b/tests/yamlparser/error_fixtures/include_missing_path.yaml @@ -0,0 +1,10 @@ +- job-template: + name: sample-job + builders: + - shell: + !include: missing-file.sh + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/include_missing_path_in_j2_yaml.error b/tests/yamlparser/error_fixtures/include_missing_path_in_j2_yaml.error new file mode 100644 index 000000000..4367d4c8a --- /dev/null +++ b/tests/yamlparser/error_fixtures/include_missing_path_in_j2_yaml.error @@ -0,0 +1,12 @@ +include_missing_path_in_j2_yaml.yaml:12:3: In project 'sample-project' + - project: + ^ +include_missing_path_in_j2_yaml.yaml:3:3: In job template 'sample-job' + - job-template: + ^ +include_missing_path_in_j2_yaml.yaml:5:15: In expanded !j2-yaml: + builders: !j2-yaml: | + ^ +:2:5: File missing-file.sh does not exist in any of include directories: .,fixtures-dir,. + !include: missing-file.sh + ^ diff --git a/tests/yamlparser/error_fixtures/include_missing_path_in_j2_yaml.yaml b/tests/yamlparser/error_fixtures/include_missing_path_in_j2_yaml.yaml new file mode 100644 index 000000000..0c12942f6 --- /dev/null +++ b/tests/yamlparser/error_fixtures/include_missing_path_in_j2_yaml.yaml @@ -0,0 +1,15 @@ +# Check for error handling inside expanded template. + +- job-template: + name: sample-job + builders: !j2-yaml: | + {# Comment lines -#} + {# added to change templated position -#} + {# of include error -#} + - shell: + !include: missing-file.sh + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/include_raw_escape_missing_path.error b/tests/yamlparser/error_fixtures/include_raw_escape_missing_path.error new file mode 100644 index 000000000..e7971efcb --- /dev/null +++ b/tests/yamlparser/error_fixtures/include_raw_escape_missing_path.error @@ -0,0 +1,9 @@ +include_raw_escape_missing_path.yaml:7:3: In project 'sample-project' + - project: + ^ +include_raw_escape_missing_path.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +include_raw_escape_missing_path.yaml:5:11: File missing-file.sh does not exist in any of include directories: .,fixtures-dir + !include-raw-escape: missing-file.sh + ^ diff --git a/tests/yamlparser/error_fixtures/include_raw_escape_missing_path.yaml b/tests/yamlparser/error_fixtures/include_raw_escape_missing_path.yaml new file mode 100644 index 000000000..7eb243df5 --- /dev/null +++ b/tests/yamlparser/error_fixtures/include_raw_escape_missing_path.yaml @@ -0,0 +1,10 @@ +- job-template: + name: sample-job + builders: + - shell: + !include-raw-escape: missing-file.sh + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/include_raw_missing_path.error b/tests/yamlparser/error_fixtures/include_raw_missing_path.error new file mode 100644 index 000000000..b85c3e88d --- /dev/null +++ b/tests/yamlparser/error_fixtures/include_raw_missing_path.error @@ -0,0 +1,9 @@ +include_raw_missing_path.yaml:7:3: In project 'sample-project' + - project: + ^ +include_raw_missing_path.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +include_raw_missing_path.yaml:5:11: File missing-file.sh does not exist in any of include directories: .,fixtures-dir + !include-raw: missing-file.sh + ^ diff --git a/tests/yamlparser/error_fixtures/include_raw_missing_path.yaml b/tests/yamlparser/error_fixtures/include_raw_missing_path.yaml new file mode 100644 index 000000000..5f0b06bb4 --- /dev/null +++ b/tests/yamlparser/error_fixtures/include_raw_missing_path.yaml @@ -0,0 +1,10 @@ +- job-template: + name: sample-job + builders: + - shell: + !include-raw: missing-file.sh + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_empty.error b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_empty.error new file mode 100644 index 000000000..109776523 --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_empty.error @@ -0,0 +1,12 @@ +incorrect_dimension_exclude_empty.yaml:6:3: In project 'sample-project' + - project: + ^ +incorrect_dimension_exclude_empty.yaml:1:3: In job template 'sample-job-{dimension}' + - job-template: + ^ +incorrect_dimension_exclude_empty.yaml:11:5: In template exclude list + exclude: + ^ +incorrect_dimension_exclude_empty.yaml:12:7: Expected a dict, but is empty: {} + - {} + ^ diff --git a/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_empty.yaml b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_empty.yaml new file mode 100644 index 000000000..eccebbaef --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_empty.yaml @@ -0,0 +1,14 @@ +- job-template: + name: 'sample-job-{dimension}' + builders: + - shell: echo {dimension} + +- project: + name: sample-project + dimension: + - first + - second + exclude: + - {} + jobs: + - 'sample-job-{dimension}' diff --git a/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_not_dict.error b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_not_dict.error new file mode 100644 index 000000000..cb8fe24e8 --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_not_dict.error @@ -0,0 +1,12 @@ +incorrect_dimension_exclude_not_dict.yaml:6:3: In project 'sample-project' + - project: + ^ +incorrect_dimension_exclude_not_dict.yaml:1:3: In job template 'sample-job-{dimension}' + - job-template: + ^ +incorrect_dimension_exclude_not_dict.yaml:11:5: In template exclude list + exclude: + ^ +incorrect_dimension_exclude_not_dict.yaml:12:7: Expected a dict, but got: 'wrong-value' + - wrong-value + ^ diff --git a/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_not_dict.yaml b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_not_dict.yaml new file mode 100644 index 000000000..4beba1c5e --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_not_dict.yaml @@ -0,0 +1,14 @@ +- job-template: + name: 'sample-job-{dimension}' + builders: + - shell: echo {dimension} + +- project: + name: sample-project + dimension: + - first + - second + exclude: + - wrong-value + jobs: + - 'sample-job-{dimension}' diff --git a/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_unknown_axis.error b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_unknown_axis.error new file mode 100644 index 000000000..f31ffa7df --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_unknown_axis.error @@ -0,0 +1,12 @@ +incorrect_dimension_exclude_unknown_axis.yaml:6:3: In project 'sample-project' + - project: + ^ +incorrect_dimension_exclude_unknown_axis.yaml:1:3: In job template 'sample-job-{dimension}' + - job-template: + ^ +incorrect_dimension_exclude_unknown_axis.yaml:11:5: In template exclude list + exclude: + ^ +incorrect_dimension_exclude_unknown_axis.yaml:13:7: Unknown axis 'wrong_axis' + - wrong_axis: some-value + ^ diff --git a/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_unknown_axis.yaml b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_unknown_axis.yaml new file mode 100644 index 000000000..589089b87 --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_dimension_exclude_unknown_axis.yaml @@ -0,0 +1,15 @@ +- job-template: + name: 'sample-job-{dimension}' + builders: + - shell: echo {dimension} + +- project: + name: sample-project + dimension: + - first + - second + exclude: + - dimension: second + - wrong_axis: some-value + jobs: + - 'sample-job-{dimension}' diff --git a/tests/yamlparser/error_fixtures/incorrect_job_spec_multi_keys.error b/tests/yamlparser/error_fixtures/incorrect_job_spec_multi_keys.error new file mode 100644 index 000000000..9021caff0 --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_job_spec_multi_keys.error @@ -0,0 +1,6 @@ +incorrect_job_spec_multi_keys.yaml:4:3: In project 'sample-project' + - project: + ^ +incorrect_job_spec_multi_keys.yaml:7:9: Job/view dict should be single-item, but have keys ['sample-job', 'incorrectly_indented_parameter']. Missing indent? + - sample-job: + ^ diff --git a/tests/yamlparser/error_fixtures/incorrect_job_spec_multi_keys.yaml b/tests/yamlparser/error_fixtures/incorrect_job_spec_multi_keys.yaml new file mode 100644 index 000000000..6cd2a758e --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_job_spec_multi_keys.yaml @@ -0,0 +1,8 @@ +- job-template: + name: sample-job + +- project: + name: sample-project + jobs: + - sample-job: + incorrectly_indented_parameter: diff --git a/tests/yamlparser/error_fixtures/incorrect_job_spec_params_not_dict.error b/tests/yamlparser/error_fixtures/incorrect_job_spec_params_not_dict.error new file mode 100644 index 000000000..a19ca532c --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_job_spec_params_not_dict.error @@ -0,0 +1,6 @@ +incorrect_job_spec_params_not_dict.yaml:4:3: In project 'sample-project' + - project: + ^ +incorrect_job_spec_params_not_dict.yaml:8:11: Job/view 'sample-job' params type should be dict, but is ['abc', 'def']. + - abc + ^ diff --git a/tests/yamlparser/error_fixtures/incorrect_job_spec_params_not_dict.yaml b/tests/yamlparser/error_fixtures/incorrect_job_spec_params_not_dict.yaml new file mode 100644 index 000000000..e6b8bb86b --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_job_spec_params_not_dict.yaml @@ -0,0 +1,9 @@ +- job-template: + name: sample-job + +- project: + name: sample-project + jobs: + - sample-job: + - abc + - def diff --git a/tests/yamlparser/error_fixtures/incorrect_job_spec_type.error b/tests/yamlparser/error_fixtures/incorrect_job_spec_type.error new file mode 100644 index 000000000..0e71365bf --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_job_spec_type.error @@ -0,0 +1,6 @@ +incorrect_job_spec_type.yaml:4:3: In project 'sample-project' + - project: + ^ +incorrect_job_spec_type.yaml:7:9: Job/view spec should name or dict, but is (123). Missing indent? + - 123 + ^ diff --git a/tests/yamlparser/error_fixtures/incorrect_job_spec_type.yaml b/tests/yamlparser/error_fixtures/incorrect_job_spec_type.yaml new file mode 100644 index 000000000..42df99e2d --- /dev/null +++ b/tests/yamlparser/error_fixtures/incorrect_job_spec_type.yaml @@ -0,0 +1,7 @@ +- job-template: + name: sample-job + +- project: + name: sample-project + jobs: + - 123 diff --git a/tests/yamlparser/error_fixtures/incorrect_template_dimensions.error b/tests/yamlparser/error_fixtures/incorrect_template_dimensions.error index 52d78ba34..df0c75ee1 100644 --- a/tests/yamlparser/error_fixtures/incorrect_template_dimensions.error +++ b/tests/yamlparser/error_fixtures/incorrect_template_dimensions.error @@ -1 +1,12 @@ -Invalid parameter 'stream' definition for template 'template-incorrect-args-{stream}-{os}': Expected a value or a dict with single element, but got: {'current': None, 'branch': 'current'} +incorrect_template_dimensions.yaml:1:3: In project 'template_incorrect_args' + - project: + ^ +incorrect_template_dimensions.yaml:14:3: In job template 'template-incorrect-args-{stream}-{os}' + - job-template: + ^ +incorrect_template_dimensions.yaml:6:5: In pamareter 'stream' definition + stream: + ^ +incorrect_template_dimensions.yaml:7:9: Expected a value or a dict with single element, but got: {'current': None, 'branch': 'current'} + - current: + ^ diff --git a/tests/yamlparser/error_fixtures/invalid_include_path_type.error b/tests/yamlparser/error_fixtures/invalid_include_path_type.error new file mode 100644 index 000000000..33702755c --- /dev/null +++ b/tests/yamlparser/error_fixtures/invalid_include_path_type.error @@ -0,0 +1,3 @@ +invalid_include_path_type.yaml:5:11: Expected either a sequence or scalar node, but found mapping + !include: + ^ diff --git a/tests/yamlparser/error_fixtures/invalid_include_path_type.yaml b/tests/yamlparser/error_fixtures/invalid_include_path_type.yaml new file mode 100644 index 000000000..3e3b0073f --- /dev/null +++ b/tests/yamlparser/error_fixtures/invalid_include_path_type.yaml @@ -0,0 +1,11 @@ +- job-template: + name: sample-job + builders: + - shell: + !include: + key: value + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/job_group_includes_missing_job.error b/tests/yamlparser/error_fixtures/job_group_includes_missing_job.error index a922ededa..d518f4bcc 100644 --- a/tests/yamlparser/error_fixtures/job_group_includes_missing_job.error +++ b/tests/yamlparser/error_fixtures/job_group_includes_missing_job.error @@ -1 +1,9 @@ -Job group group-1: Failed to find suitable job/view/template named 'job-2' +job_group_includes_missing_job.yaml:14:3: In project 'sample-project' + - project: + ^ +job_group_includes_missing_job.yaml:8:3: In job group 'group-1' + - job-group: + ^ +job_group_includes_missing_job.yaml:12:9: Failed to find suitable job/view/template named 'job-2' + - job-2 + ^ diff --git a/tests/yamlparser/error_fixtures/missing_defaults_at_job.error b/tests/yamlparser/error_fixtures/missing_defaults_at_job.error new file mode 100644 index 000000000..cd5436ab2 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_defaults_at_job.error @@ -0,0 +1,6 @@ +missing_defaults_at_job.yaml:1:3: In job 'sample-job' + - job: + ^ +missing_defaults_at_job.yaml:3:15: Job 'sample-job' wants defaults 'missing-defaults' but it was never defined + defaults: missing-defaults + ^ diff --git a/tests/yamlparser/error_fixtures/missing_defaults_at_job.yaml b/tests/yamlparser/error_fixtures/missing_defaults_at_job.yaml new file mode 100644 index 000000000..e883ef41e --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_defaults_at_job.yaml @@ -0,0 +1,8 @@ +- job: + name: sample-job + defaults: missing-defaults + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/missing_defaults_at_job_template.error b/tests/yamlparser/error_fixtures/missing_defaults_at_job_template.error new file mode 100644 index 000000000..2b56f7646 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_defaults_at_job_template.error @@ -0,0 +1,9 @@ +missing_defaults_at_job_template.yaml:5:3: In project 'sample-project' + - project: + ^ +missing_defaults_at_job_template.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +missing_defaults_at_job_template.yaml:3:15: Job template 'sample-job' wants defaults 'missing-defaults' but it was never defined + defaults: missing-defaults + ^ diff --git a/tests/yamlparser/error_fixtures/missing_defaults_at_job_template.yaml b/tests/yamlparser/error_fixtures/missing_defaults_at_job_template.yaml new file mode 100644 index 000000000..9c0b3ecad --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_defaults_at_job_template.yaml @@ -0,0 +1,8 @@ +- job-template: + name: sample-job + defaults: missing-defaults + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/missing_defaults_at_project.error b/tests/yamlparser/error_fixtures/missing_defaults_at_project.error new file mode 100644 index 000000000..1d58bad72 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_defaults_at_project.error @@ -0,0 +1,9 @@ +missing_defaults_at_project.yaml:4:3: In project 'sample-project' + - project: + ^ +missing_defaults_at_project.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +missing_defaults_at_project.yaml:6:15: Job template 'sample-job' wants defaults 'global' but it was never defined + defaults: missing-defaults + ^ diff --git a/tests/yamlparser/error_fixtures/missing_defaults_at_project.yaml b/tests/yamlparser/error_fixtures/missing_defaults_at_project.yaml new file mode 100644 index 000000000..2a5335d5d --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_defaults_at_project.yaml @@ -0,0 +1,8 @@ +- job-template: + name: sample-job + +- project: + name: sample-project + defaults: missing-defaults + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/missing_job_element_name.error b/tests/yamlparser/error_fixtures/missing_job_element_name.error new file mode 100644 index 000000000..d4a3f87b6 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_job_element_name.error @@ -0,0 +1,3 @@ +missing_job_element_name.yaml:3:5: Missing required element: 'name' + some_thing: value + ^ diff --git a/tests/yamlparser/error_fixtures/missing_job_element_name.yaml b/tests/yamlparser/error_fixtures/missing_job_element_name.yaml new file mode 100644 index 000000000..3e822bc76 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_job_element_name.yaml @@ -0,0 +1,3 @@ +# Job with no 'name' element defined. +- job: + some_thing: value diff --git a/tests/yamlparser/error_fixtures/missing_macro_element.error b/tests/yamlparser/error_fixtures/missing_macro_element.error new file mode 100644 index 000000000..b9439bea8 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_macro_element.error @@ -0,0 +1,3 @@ +missing_macro_element.yaml:3:5: Missing required element: 'builders' + name: sample-builder + ^ diff --git a/tests/yamlparser/error_fixtures/missing_macro_element.yaml b/tests/yamlparser/error_fixtures/missing_macro_element.yaml new file mode 100644 index 000000000..4097b5228 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_macro_element.yaml @@ -0,0 +1,3 @@ +# Builder macro with no 'builders' element defined. +- builder: + name: sample-builder diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.error b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.error new file mode 100644 index 000000000..0b332a6f3 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.error @@ -0,0 +1,15 @@ +missing_param_in_include_jinja2.yaml:6:3: In project 'sample-project' + - project: + ^ +missing_param_in_include_jinja2.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +missing_param_in_include_jinja2.yaml:4:16: In included file 'missing_param_in_include_jinja2.inc' + - shell: !include-jinja2: missing_param_in_include_jin ... + ^ +missing_param_in_include_jinja2.yaml:4:16: While formatting jinja2 template '{# Sample comment #}\n#!/bin/bash\n\nif [ -...' + - shell: !include-jinja2: missing_param_in_include_jin ... + ^ +missing_param_in_include_jinja2.inc:5:5: 'johnny' is undefined + echo "Here is {{ johnny }}!" + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.inc b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.inc new file mode 100644 index 000000000..ffc69b36a --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.inc @@ -0,0 +1,6 @@ +{# Sample comment #} +#!/bin/bash + +if [ -f the-door ]; then + echo "Here is {{ johnny }}!" +fi diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.yaml b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.yaml new file mode 100644 index 000000000..f242dc172 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.yaml @@ -0,0 +1,9 @@ +- job-template: + name: sample-job + builders: + - shell: !include-jinja2: missing_param_in_include_jinja2.inc + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.error b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.error new file mode 100644 index 000000000..db043ba38 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.error @@ -0,0 +1,15 @@ +missing_param_in_include_jinja2_format.yaml:5:3: In project 'sample-project' + - project: + ^ +missing_param_in_include_jinja2_format.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +missing_param_in_include_jinja2_format.yaml:3:15: In included file 'missing_param_in_include_jinja2_format.yaml.inc' + builders: !include: missing_param_in_include_jinja2_for ... + ^ +missing_param_in_include_jinja2_format.yaml.inc:2:14: While formatting jinja2 template '#!/bin/bash\n\necho "hello, {{ unknown_one...' + command: !j2: | + ^ +missing_param_in_include_jinja2_format.yaml.inc:5:7: 'unknown_one' is undefined + echo "hello, {{ unknown_one }}!" + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.yaml b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.yaml new file mode 100644 index 000000000..3f4f15dc1 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.yaml @@ -0,0 +1,8 @@ +- job-template: + name: sample-job + builders: !include: missing_param_in_include_jinja2_format.yaml.inc + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.yaml.inc b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.yaml.inc new file mode 100644 index 000000000..a5ec5f955 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.yaml.inc @@ -0,0 +1,5 @@ +- shell: + command: !j2: | + #!/bin/bash + + echo "hello, {{ unknown_one }}!" diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_raw.error b/tests/yamlparser/error_fixtures/missing_param_in_include_raw.error new file mode 100644 index 000000000..1843b3a27 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_raw.error @@ -0,0 +1,10 @@ +missing_param_in_include_raw.yaml:6:3: In project 'sample-project' + - project: + ^ +missing_param_in_include_raw.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +missing_param_in_include_raw.yaml:4:16: In included file 'missing_param_in_include_raw.inc' + - shell: !include-raw: missing_param_in_include_raw.inc + ^ +While formatting string '#!/bin/bash\necho "This one is {missing}!"\n...': Missing parameter: 'missing' diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_raw.inc b/tests/yamlparser/error_fixtures/missing_param_in_include_raw.inc new file mode 100644 index 000000000..0ae93b30b --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_raw.inc @@ -0,0 +1,2 @@ +#!/bin/bash +echo "This one is {missing}!" diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_raw.yaml b/tests/yamlparser/error_fixtures/missing_param_in_include_raw.yaml new file mode 100644 index 000000000..4062d7680 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_raw.yaml @@ -0,0 +1,9 @@ +- job-template: + name: sample-job + builders: + - shell: !include-raw: missing_param_in_include_raw.inc + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_simple_format.error b/tests/yamlparser/error_fixtures/missing_param_in_include_simple_format.error new file mode 100644 index 000000000..2ce340223 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_simple_format.error @@ -0,0 +1,12 @@ +missing_param_in_include_simple_format.yaml:5:3: In project 'sample-project' + - project: + ^ +missing_param_in_include_simple_format.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +missing_param_in_include_simple_format.yaml:3:15: In included file 'missing_param_in_include_simple_format.yaml.inc' + builders: !include: missing_param_in_include_simple_for ... + ^ +missing_param_in_include_simple_format.yaml.inc:3:7: While formatting string '#!/bin/bash\n\necho "hello, {unknown_one}!"\n...': Missing parameter: 'unknown_one' + #!/bin/bash + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_simple_format.yaml b/tests/yamlparser/error_fixtures/missing_param_in_include_simple_format.yaml new file mode 100644 index 000000000..b78725b5e --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_simple_format.yaml @@ -0,0 +1,8 @@ +- job-template: + name: sample-job + builders: !include: missing_param_in_include_simple_format.yaml.inc + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/missing_param_in_include_simple_format.yaml.inc b/tests/yamlparser/error_fixtures/missing_param_in_include_simple_format.yaml.inc new file mode 100644 index 000000000..7ae476d0a --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_include_simple_format.yaml.inc @@ -0,0 +1,5 @@ +- shell: + command: | + #!/bin/bash + + echo "hello, {unknown_one}!" diff --git a/tests/yamlparser/error_fixtures/missing_param_in_j2_yaml.error b/tests/yamlparser/error_fixtures/missing_param_in_j2_yaml.error new file mode 100644 index 000000000..0d33e03fa --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_j2_yaml.error @@ -0,0 +1,12 @@ +missing_param_in_j2_yaml.yaml:10:3: In project 'sample-project' + - project: + ^ +missing_param_in_j2_yaml.yaml:1:3: In job template 'sample-job' + - job-template: + ^ +missing_param_in_j2_yaml.yaml:3:15: While formatting jinja2 template '- shell:\n command: |\n #!/bin/bas...' + builders: !j2-yaml: | + ^ +missing_param_in_j2_yaml.yaml:8:13: 'unknown_one' is undefined + echo "hello, {{ unknown_one }}!" + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_in_j2_yaml.yaml b/tests/yamlparser/error_fixtures/missing_param_in_j2_yaml.yaml new file mode 100644 index 000000000..d1037d686 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_in_j2_yaml.yaml @@ -0,0 +1,13 @@ +- job-template: + name: sample-job + builders: !j2-yaml: | + - shell: + command: | + #!/bin/bash + + echo "hello, {{ unknown_one }}!" + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_job_direct_single_line.error b/tests/yamlparser/error_fixtures/missing_param_jinja2_job_direct_single_line.error new file mode 100644 index 000000000..59d44647b --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_jinja2_job_direct_single_line.error @@ -0,0 +1,9 @@ +missing_param_jinja2_job_direct_single_line.yaml:1:3: In job 'sample-job' + - job: + ^ +missing_param_jinja2_job_direct_single_line.yaml:4:16: While formatting jinja2 template 'echo {{ missing_param }}' + - shell: !j2: echo {{ missing_param }} + ^ +missing_param_jinja2_job_direct_single_line.yaml:4:21: 'missing_param' is undefined + - shell: !j2: echo {{ missing_param }} + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_job_direct_single_line.yaml b/tests/yamlparser/error_fixtures/missing_param_jinja2_job_direct_single_line.yaml new file mode 100644 index 000000000..b08bfbe84 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_jinja2_job_direct_single_line.yaml @@ -0,0 +1,4 @@ +- job: + name: sample-job + builders: + - shell: !j2: echo {{ missing_param }} diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_macro_direct.error b/tests/yamlparser/error_fixtures/missing_param_jinja2_macro_direct.error index fed29923e..cb16ff283 100644 --- a/tests/yamlparser/error_fixtures/missing_param_jinja2_macro_direct.error +++ b/tests/yamlparser/error_fixtures/missing_param_jinja2_macro_direct.error @@ -1 +1,18 @@ -While expanding macro 'sample-builder': While formatting jinja2 template 'echo {{ missing_param }} {{ other_param ...': 'missing_param' is undefined +missing_param_jinja2_macro_direct.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_jinja2_macro_direct.yaml:4:9: Defined here + - sample-job + ^ +missing_param_jinja2_macro_direct.yaml:12:3: In job template 'sample-job' + - job-template: + ^ +missing_param_jinja2_macro_direct.yaml:6:3: While expanding macro 'sample-builder' + - builder: + ^ +missing_param_jinja2_macro_direct.yaml:9:16: While formatting jinja2 template 'echo {{ missing_param }} {{ other_param ...' + - shell: !j2: | + ^ +missing_param_jinja2_macro_direct.yaml:10:11: 'missing_param' is undefined + echo {{ missing_param }} {{ other_param }} + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_macro_indirect.error b/tests/yamlparser/error_fixtures/missing_param_jinja2_macro_indirect.error index 1e7beae68..966893121 100644 --- a/tests/yamlparser/error_fixtures/missing_param_jinja2_macro_indirect.error +++ b/tests/yamlparser/error_fixtures/missing_param_jinja2_macro_indirect.error @@ -1 +1,21 @@ -While expanding 'param_1', used by 'param_3', used by'param_2', used by template 'sample-job': While formatting jinja2 template '{{ missing_param }}': 'missing_param' is undefined +missing_param_jinja2_macro_indirect.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_jinja2_macro_indirect.yaml:16:3: In job template 'sample-job' + - job-template: + ^ +missing_param_jinja2_macro_indirect.yaml:18:14: Used by param_3 + param_3: '{param_2}-plus' + ^ +missing_param_jinja2_macro_indirect.yaml:4:14: Used by param_2 + param_2: '{param_1}' + ^ +missing_param_jinja2_macro_indirect.yaml:3:5: While expanding parameter 'param_1' + param_1: !j2: '{{ missing_param }}' + ^ +missing_param_jinja2_macro_indirect.yaml:3:14: While formatting jinja2 template '{{ missing_param }}' + param_1: !j2: '{{ missing_param }}' + ^ +missing_param_jinja2_macro_indirect.yaml:3:20: 'missing_param' is undefined + param_1: !j2: '{{ missing_param }}' + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct.error b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct.error deleted file mode 100644 index 8efe41b73..000000000 --- a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct.error +++ /dev/null @@ -1 +0,0 @@ -While formatting jinja2 template 'echo {{ missing_param }}\n': 'missing_param' is undefined diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_multi_line.error b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_multi_line.error new file mode 100644 index 000000000..65692a171 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_multi_line.error @@ -0,0 +1,12 @@ +missing_param_jinja2_template_direct_multi_line.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_jinja2_template_direct_multi_line.yaml:6:3: In job template 'sample-job' + - job-template: + ^ +missing_param_jinja2_template_direct_multi_line.yaml:9:16: While formatting jinja2 template '#!/bin/bash\n\nset -x\necho {{ missing_para...' + - shell: !j2: | + ^ +missing_param_jinja2_template_direct_multi_line.yaml:13:11: 'missing_param' is undefined + echo {{ missing_param }} + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_multi_line.yaml b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_multi_line.yaml new file mode 100644 index 000000000..612ab10fa --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_multi_line.yaml @@ -0,0 +1,16 @@ +- project: + name: sample-project + jobs: + - sample-job + +- job-template: + name: sample-job + builders: + - shell: !j2: | + #!/bin/bash + + set -x + echo {{ missing_param }} + echo "All is done" + + exit 0 diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_quoted.error b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_quoted.error new file mode 100644 index 000000000..3079ec22b --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_quoted.error @@ -0,0 +1,12 @@ +missing_param_jinja2_template_direct_quoted.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_jinja2_template_direct_quoted.yaml:6:3: In job template 'sample-job' + - job-template: + ^ +missing_param_jinja2_template_direct_quoted.yaml:9:16: While formatting jinja2 template 'echo {{ missing_param }}' + - shell: !j2: 'echo {{ missing_param }}' + ^ +missing_param_jinja2_template_direct_quoted.yaml:9:22: 'missing_param' is undefined + - shell: !j2: 'echo {{ missing_param }}' + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_quoted.yaml b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_quoted.yaml new file mode 100644 index 000000000..1e6ed1bbc --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_quoted.yaml @@ -0,0 +1,9 @@ +- project: + name: sample-project + jobs: + - sample-job + +- job-template: + name: sample-job + builders: + - shell: !j2: 'echo {{ missing_param }}' diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_single_line.error b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_single_line.error new file mode 100644 index 000000000..bad4cbb07 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_single_line.error @@ -0,0 +1,12 @@ +missing_param_jinja2_template_direct_single_line.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_jinja2_template_direct_single_line.yaml:6:3: In job template 'sample-job' + - job-template: + ^ +missing_param_jinja2_template_direct_single_line.yaml:9:16: While formatting jinja2 template 'echo {{ missing_param }}' + - shell: !j2: echo {{ missing_param }} + ^ +missing_param_jinja2_template_direct_single_line.yaml:9:21: 'missing_param' is undefined + - shell: !j2: echo {{ missing_param }} + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_single_line.yaml b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_single_line.yaml new file mode 100644 index 000000000..2848e0f4a --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct_single_line.yaml @@ -0,0 +1,9 @@ +- project: + name: sample-project + jobs: + - sample-job + +- job-template: + name: sample-job + builders: + - shell: !j2: echo {{ missing_param }} diff --git a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_indirect.error b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_indirect.error index 1e7beae68..9bdcd13d0 100644 --- a/tests/yamlparser/error_fixtures/missing_param_jinja2_template_indirect.error +++ b/tests/yamlparser/error_fixtures/missing_param_jinja2_template_indirect.error @@ -1 +1,21 @@ -While expanding 'param_1', used by 'param_3', used by'param_2', used by template 'sample-job': While formatting jinja2 template '{{ missing_param }}': 'missing_param' is undefined +missing_param_jinja2_template_indirect.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_jinja2_template_indirect.yaml:8:3: In job template 'sample-job' + - job-template: + ^ +missing_param_jinja2_template_indirect.yaml:10:14: Used by param_3 + param_3: '{param_2}-plus' + ^ +missing_param_jinja2_template_indirect.yaml:4:14: Used by param_2 + param_2: '{param_1}' + ^ +missing_param_jinja2_template_indirect.yaml:3:5: While expanding parameter 'param_1' + param_1: !j2: '{{ missing_param }}' + ^ +missing_param_jinja2_template_indirect.yaml:3:14: While formatting jinja2 template '{{ missing_param }}' + param_1: !j2: '{{ missing_param }}' + ^ +missing_param_jinja2_template_indirect.yaml:3:20: 'missing_param' is undefined + param_1: !j2: '{{ missing_param }}' + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_simple_macro_direct.error b/tests/yamlparser/error_fixtures/missing_param_simple_macro_direct.error index b1bdbe6e7..454ca3a13 100644 --- a/tests/yamlparser/error_fixtures/missing_param_simple_macro_direct.error +++ b/tests/yamlparser/error_fixtures/missing_param_simple_macro_direct.error @@ -1 +1,15 @@ -While expanding macro 'sample-builder': While formatting string 'echo {missing_param} {other_param}\n': Missing parameter: 'missing_param' +missing_param_simple_macro_direct.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_simple_macro_direct.yaml:4:9: Defined here + - sample-job + ^ +missing_param_simple_macro_direct.yaml:12:3: In job template 'sample-job' + - job-template: + ^ +missing_param_simple_macro_direct.yaml:6:3: While expanding macro 'sample-builder' + - builder: + ^ +missing_param_simple_macro_direct.yaml:10:11: While formatting string 'echo {missing_param} {other_param}\n': Missing parameter: 'missing_param' + echo {missing_param} {other_param} + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_simple_macro_indirect.error b/tests/yamlparser/error_fixtures/missing_param_simple_macro_indirect.error index 4fd9fa66b..95b58303b 100644 --- a/tests/yamlparser/error_fixtures/missing_param_simple_macro_indirect.error +++ b/tests/yamlparser/error_fixtures/missing_param_simple_macro_indirect.error @@ -1 +1,18 @@ -While expanding 'param_1', used by 'param_3', used by'param_2', used by template 'sample-job': While formatting string '{missing_param}': 'missing_param' is undefined +missing_param_simple_macro_indirect.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_simple_macro_indirect.yaml:16:3: In job template 'sample-job' + - job-template: + ^ +missing_param_simple_macro_indirect.yaml:18:14: Used by param_3 + param_3: '{param_2}-plus' + ^ +missing_param_simple_macro_indirect.yaml:4:14: Used by param_2 + param_2: '{param_1}' + ^ +missing_param_simple_macro_indirect.yaml:3:5: While expanding parameter 'param_1' + param_1: '{missing_param}' + ^ +missing_param_simple_macro_indirect.yaml:3:15: While formatting string '{missing_param}': 'missing_param' is undefined + param_1: '{missing_param}' + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_simple_template_direct.error b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct.error deleted file mode 100644 index f70d1e577..000000000 --- a/tests/yamlparser/error_fixtures/missing_param_simple_template_direct.error +++ /dev/null @@ -1 +0,0 @@ -While formatting string 'echo {missing_param}\n': Missing parameter: 'missing_param' diff --git a/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_multi_line.error b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_multi_line.error new file mode 100644 index 000000000..09221031f --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_multi_line.error @@ -0,0 +1,9 @@ +missing_param_simple_template_direct_multi_line.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_simple_template_direct_multi_line.yaml:6:3: In job template 'sample-job' + - job-template: + ^ +missing_param_simple_template_direct_multi_line.yaml:10:11: While formatting string '#!/bin/bash\necho {missing_param}\n': Missing parameter: 'missing_param' + #!/bin/bash + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_simple_template_direct.yaml b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_multi_line.yaml similarity index 88% rename from tests/yamlparser/error_fixtures/missing_param_simple_template_direct.yaml rename to tests/yamlparser/error_fixtures/missing_param_simple_template_direct_multi_line.yaml index b95805fcc..83508c5b4 100644 --- a/tests/yamlparser/error_fixtures/missing_param_simple_template_direct.yaml +++ b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_multi_line.yaml @@ -7,4 +7,5 @@ name: sample-job builders: - shell: | + #!/bin/bash echo {missing_param} diff --git a/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_quoted.error b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_quoted.error new file mode 100644 index 000000000..364461650 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_quoted.error @@ -0,0 +1,9 @@ +missing_param_simple_template_direct_quoted.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_simple_template_direct_quoted.yaml:6:3: In job template 'sample-job' + - job-template: + ^ +missing_param_simple_template_direct_quoted.yaml:9:17: While formatting string 'echo {missing_param}': Missing parameter: 'missing_param' + - shell: 'echo {missing_param}' + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_quoted.yaml b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_quoted.yaml new file mode 100644 index 000000000..1cd5d79fc --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_quoted.yaml @@ -0,0 +1,9 @@ +- project: + name: sample-project + jobs: + - sample-job + +- job-template: + name: sample-job + builders: + - shell: 'echo {missing_param}' diff --git a/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_single_line.error b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_single_line.error new file mode 100644 index 000000000..3d1764ec3 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_single_line.error @@ -0,0 +1,9 @@ +missing_param_simple_template_direct_single_line.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_simple_template_direct_single_line.yaml:6:3: In job template 'sample-job' + - job-template: + ^ +missing_param_simple_template_direct_single_line.yaml:9:16: While formatting string 'echo {missing_param}': Missing parameter: 'missing_param' + - shell: echo {missing_param} + ^ diff --git a/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_single_line.yaml b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_single_line.yaml new file mode 100644 index 000000000..dba5cbd59 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_param_simple_template_direct_single_line.yaml @@ -0,0 +1,9 @@ +- project: + name: sample-project + jobs: + - sample-job + +- job-template: + name: sample-job + builders: + - shell: echo {missing_param} diff --git a/tests/yamlparser/error_fixtures/missing_param_simple_template_indirect.error b/tests/yamlparser/error_fixtures/missing_param_simple_template_indirect.error index e86a2b5b8..ab8be61e8 100644 --- a/tests/yamlparser/error_fixtures/missing_param_simple_template_indirect.error +++ b/tests/yamlparser/error_fixtures/missing_param_simple_template_indirect.error @@ -1 +1,15 @@ -While expanding 'param_1', used by 'param_3', used by template 'sample-job': While formatting string '{missing_param}': 'missing_param' is undefined +missing_param_simple_template_indirect.yaml:1:3: In project 'sample-project' + - project: + ^ +missing_param_simple_template_indirect.yaml:8:3: In job template 'sample-job' + - job-template: + ^ +missing_param_simple_template_indirect.yaml:10:14: Used by param_3 + param_3: '{param_1}-plus' + ^ +missing_param_simple_template_indirect.yaml:3:5: While expanding parameter 'param_1' + param_1: '{missing_param}' + ^ +missing_param_simple_template_indirect.yaml:3:15: While formatting string '{missing_param}': 'missing_param' is undefined + param_1: '{missing_param}' + ^ diff --git a/tests/yamlparser/error_fixtures/missing_project_element_name.error b/tests/yamlparser/error_fixtures/missing_project_element_name.error new file mode 100644 index 000000000..516c76d87 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_project_element_name.error @@ -0,0 +1,3 @@ +missing_project_element_name.yaml:3:5: Missing required element: 'name' + some_param: value + ^ diff --git a/tests/yamlparser/error_fixtures/missing_project_element_name.yaml b/tests/yamlparser/error_fixtures/missing_project_element_name.yaml new file mode 100644 index 000000000..d7a9c38df --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_project_element_name.yaml @@ -0,0 +1,3 @@ +# Project with no 'name' element defined. +- project: + some_param: value diff --git a/tests/yamlparser/error_fixtures/parameter_name_reuse_default.error b/tests/yamlparser/error_fixtures/parameter_name_reuse_default.error index 09080e9fc..a8b90696d 100644 --- a/tests/yamlparser/error_fixtures/parameter_name_reuse_default.error +++ b/tests/yamlparser/error_fixtures/parameter_name_reuse_default.error @@ -1 +1,15 @@ -While expanding 'timer' for template 'sample-job': Recursive parameters usage: timer <- timer +parameter_name_reuse_default.yaml:10:3: In project 'sample-project' + - project: + ^ +parameter_name_reuse_default.yaml:3:3: In job template 'sample-job' + - job-template: + ^ +parameter_name_reuse_default.yaml:5:12: Used by timer + timer: '{timer}' + ^ +parameter_name_reuse_default.yaml:5:5: While expanding 'timer' + timer: '{timer}' + ^ +parameter_name_reuse_default.yaml:5:12: Recursive parameters usage: timer + timer: '{timer}' + ^ diff --git a/tests/yamlparser/error_fixtures/parameter_name_reuse_group_override.error b/tests/yamlparser/error_fixtures/parameter_name_reuse_group_override.error index 09d45ee45..cf859e21b 100644 --- a/tests/yamlparser/error_fixtures/parameter_name_reuse_group_override.error +++ b/tests/yamlparser/error_fixtures/parameter_name_reuse_group_override.error @@ -1 +1,18 @@ -While expanding 'param' for template 'sample-job': Recursive parameters usage: param <- param +parameter_name_reuse_group_override.yaml:18:3: In project 'sample-project' + - project: + ^ +parameter_name_reuse_group_override.yaml:12:3: In job group 'sample-group' + - job-group: + ^ +parameter_name_reuse_group_override.yaml:5:3: In job template 'sample-job' + - job-template: + ^ +parameter_name_reuse_group_override.yaml:16:18: Used by param + param: '{param}-a' + ^ +parameter_name_reuse_group_override.yaml:16:11: While expanding 'param' + param: '{param}-a' + ^ +parameter_name_reuse_group_override.yaml:16:18: Recursive parameters usage: param + param: '{param}-a' + ^ diff --git a/tests/yamlparser/error_fixtures/project_includes_missing_job.error b/tests/yamlparser/error_fixtures/project_includes_missing_job.error index 7338029e9..ac1776979 100644 --- a/tests/yamlparser/error_fixtures/project_includes_missing_job.error +++ b/tests/yamlparser/error_fixtures/project_includes_missing_job.error @@ -1 +1,6 @@ -Project sample-project: Failed to find suitable job/view/template named 'job-2' +project_includes_missing_job.yaml:8:3: In project 'sample-project' + - project: + ^ +project_includes_missing_job.yaml:12:9: Failed to find suitable job/view/template named 'job-2' + - job-2: + ^ diff --git a/tests/yamlparser/error_fixtures/project_includes_missing_job.yaml b/tests/yamlparser/error_fixtures/project_includes_missing_job.yaml index 8414f4a79..2471a3279 100644 --- a/tests/yamlparser/error_fixtures/project_includes_missing_job.yaml +++ b/tests/yamlparser/error_fixtures/project_includes_missing_job.yaml @@ -9,4 +9,5 @@ name: sample-project jobs: - job-1 - - job-2 + - job-2: + some_param: 123 diff --git a/tests/yamlparser/error_fixtures/project_includes_missing_view.error b/tests/yamlparser/error_fixtures/project_includes_missing_view.error index 551724d05..b23c6526f 100644 --- a/tests/yamlparser/error_fixtures/project_includes_missing_view.error +++ b/tests/yamlparser/error_fixtures/project_includes_missing_view.error @@ -1 +1,6 @@ -Project sample-project: Failed to find suitable job/view/template named 'view-2' +project_includes_missing_view.yaml:7:3: In project 'sample-project' + - project: + ^ +project_includes_missing_view.yaml:11:9: Failed to find suitable job/view/template named 'view-2' + - view-2 + ^ diff --git a/tests/yamlparser/error_fixtures/recursive_parameter.error b/tests/yamlparser/error_fixtures/recursive_parameter.error new file mode 100644 index 000000000..9a931f709 --- /dev/null +++ b/tests/yamlparser/error_fixtures/recursive_parameter.error @@ -0,0 +1,24 @@ +recursive_parameter.yaml:11:3: In project 'sample-project' + - project: + ^ +recursive_parameter.yaml:7:3: In job template 'sample-job-{param_1}' + - job-template: + ^ +recursive_parameter.yaml:5:9: Used by param_1 + - '{param_2}-at-project' + ^ +recursive_parameter.yaml:9:14: Used by param_2 + param_2: '{param_3}-at-template' + ^ +recursive_parameter.yaml:13:14: Used by param_3 + param_3: '{param_4}-at-globals' + ^ +recursive_parameter.yaml:16:20: Used by param_4 + param_4: '{param_1}-at-job-spec' + ^ +recursive_parameter.yaml:3:5: While expanding 'param_1' + param_1: + ^ +recursive_parameter.yaml:5:9: Recursive parameters usage: param_1 <- param_2 <- param_3 <- param_4 + - '{param_2}-at-project' + ^ diff --git a/tests/yamlparser/error_fixtures/recursive_parameter.yaml b/tests/yamlparser/error_fixtures/recursive_parameter.yaml new file mode 100644 index 000000000..6596aadce --- /dev/null +++ b/tests/yamlparser/error_fixtures/recursive_parameter.yaml @@ -0,0 +1,16 @@ +- defaults: + name: global + param_1: + - param_1_value_1 + - '{param_2}-at-project' + +- job-template: + name: 'sample-job-{param_1}' + param_2: '{param_3}-at-template' + +- project: + name: sample-project + param_3: '{param_4}-at-globals' + jobs: + - 'sample-job-{param_1}': + param_4: '{param_1}-at-job-spec' diff --git a/tests/yamlparser/error_fixtures/string_join_not_2_elements.error b/tests/yamlparser/error_fixtures/string_join_not_2_elements.error new file mode 100644 index 000000000..8b6176394 --- /dev/null +++ b/tests/yamlparser/error_fixtures/string_join_not_2_elements.error @@ -0,0 +1,3 @@ +string_join_not_2_elements.yaml:4:16: Join value should contain 2 elements: delimiter and string list, but contains 1 elements: ['element_1'] + - shell: !join: + ^ diff --git a/tests/yamlparser/error_fixtures/string_join_not_2_elements.yaml b/tests/yamlparser/error_fixtures/string_join_not_2_elements.yaml new file mode 100644 index 000000000..7528d6833 --- /dev/null +++ b/tests/yamlparser/error_fixtures/string_join_not_2_elements.yaml @@ -0,0 +1,5 @@ +- job-template: + name: sample-job + builders: + - shell: !join: + - element_1 diff --git a/tests/yamlparser/error_fixtures/topmost_collection_not_a_list.error b/tests/yamlparser/error_fixtures/topmost_collection_not_a_list.error new file mode 100644 index 000000000..d3478a161 --- /dev/null +++ b/tests/yamlparser/error_fixtures/topmost_collection_not_a_list.error @@ -0,0 +1,3 @@ +topmost_collection_not_a_list.yaml:2:1: The topmost collection must be a list, but is: {'key_1': 'value-1', 'key_2': 'value-2'} + key_1: value-1 + ^ diff --git a/tests/yamlparser/error_fixtures/topmost_collection_not_a_list.yaml b/tests/yamlparser/error_fixtures/topmost_collection_not_a_list.yaml new file mode 100644 index 000000000..ea9b6866c --- /dev/null +++ b/tests/yamlparser/error_fixtures/topmost_collection_not_a_list.yaml @@ -0,0 +1,3 @@ +# Top-most collection should be a list, but here it is a dict. +key_1: value-1 +key_2: value-2 diff --git a/tests/yamlparser/error_fixtures/topmost_list_dict_has_multiple_keys.error b/tests/yamlparser/error_fixtures/topmost_list_dict_has_multiple_keys.error new file mode 100644 index 000000000..2d38e262b --- /dev/null +++ b/tests/yamlparser/error_fixtures/topmost_list_dict_has_multiple_keys.error @@ -0,0 +1,3 @@ +topmost_list_dict_has_multiple_keys.yaml:2:3: Topmost dict should be single-item, but have keys ['job', 'name']. Missing indent? + - job: + ^ diff --git a/tests/yamlparser/error_fixtures/topmost_list_dict_has_multiple_keys.yaml b/tests/yamlparser/error_fixtures/topmost_list_dict_has_multiple_keys.yaml new file mode 100644 index 000000000..70bd9b450 --- /dev/null +++ b/tests/yamlparser/error_fixtures/topmost_list_dict_has_multiple_keys.yaml @@ -0,0 +1,3 @@ +# Missing indent for job elements, resulting in dict with several keys. +- job: + name: sample-job diff --git a/tests/yamlparser/error_fixtures/topmost_list_item_not_a_dict.error b/tests/yamlparser/error_fixtures/topmost_list_item_not_a_dict.error new file mode 100644 index 000000000..3860bbf04 --- /dev/null +++ b/tests/yamlparser/error_fixtures/topmost_list_item_not_a_dict.error @@ -0,0 +1,3 @@ +topmost_list_item_not_a_dict.yaml:2:3: Topmost list should contain single-item dict, not a . Missing indent? + - some_string + ^ diff --git a/tests/yamlparser/error_fixtures/topmost_list_item_not_a_dict.yaml b/tests/yamlparser/error_fixtures/topmost_list_item_not_a_dict.yaml new file mode 100644 index 000000000..ced1fd539 --- /dev/null +++ b/tests/yamlparser/error_fixtures/topmost_list_item_not_a_dict.yaml @@ -0,0 +1,2 @@ +# Top-most collection element should be a dict, but here it is a string. +- some_string diff --git a/tests/yamlparser/error_fixtures/undefined_parameter.error b/tests/yamlparser/error_fixtures/undefined_parameter.error new file mode 100644 index 000000000..a8fcc03c9 --- /dev/null +++ b/tests/yamlparser/error_fixtures/undefined_parameter.error @@ -0,0 +1,18 @@ +undefined_parameter.yaml:11:3: In project 'sample-project' + - project: + ^ +undefined_parameter.yaml:7:3: In job template 'sample-job-{param_1}' + - job-template: + ^ +undefined_parameter.yaml:9:14: Used by param_2 + param_2: '{param_3}-at-template' + ^ +undefined_parameter.yaml:13:14: Used by param_3 + param_3: '{param_4}-at-globals' + ^ +undefined_parameter.yaml:14:5: While expanding parameter 'param_4' + param_4: '{missing_param}-error' + ^ +undefined_parameter.yaml:14:15: While formatting string '{missing_param}-error': 'missing_param' is undefined + param_4: '{missing_param}-error' + ^ diff --git a/tests/yamlparser/error_fixtures/undefined_parameter.yaml b/tests/yamlparser/error_fixtures/undefined_parameter.yaml new file mode 100644 index 000000000..be5649b6a --- /dev/null +++ b/tests/yamlparser/error_fixtures/undefined_parameter.yaml @@ -0,0 +1,16 @@ +- defaults: + name: global + param_1: + - param_1_value_1 + - '{param_2}-at-project' + +- job-template: + name: 'sample-job-{param_1}' + param_2: '{param_3}-at-template' + +- project: + name: sample-project + param_3: '{param_4}-at-globals' + param_4: '{missing_param}-error' + jobs: + - 'sample-job-{param_1}' diff --git a/tests/yamlparser/error_fixtures/unexpected_macro_elements.error b/tests/yamlparser/error_fixtures/unexpected_macro_elements.error new file mode 100644 index 000000000..aa74010b1 --- /dev/null +++ b/tests/yamlparser/error_fixtures/unexpected_macro_elements.error @@ -0,0 +1,3 @@ +unexpected_macro_elements.yaml:3:5: In builder macro 'sample-builder': unexpected elements: something_unexpected + something_unexpected: sample-value + ^ diff --git a/tests/yamlparser/error_fixtures/unexpected_macro_elements.yaml b/tests/yamlparser/error_fixtures/unexpected_macro_elements.yaml new file mode 100644 index 000000000..26ef52146 --- /dev/null +++ b/tests/yamlparser/error_fixtures/unexpected_macro_elements.yaml @@ -0,0 +1,4 @@ +- builder: + name: sample-builder + something_unexpected: sample-value + builders: [] diff --git a/tests/yamlparser/error_fixtures/unknown_topmost_element_type.error b/tests/yamlparser/error_fixtures/unknown_topmost_element_type.error new file mode 100644 index 000000000..723a91ee1 --- /dev/null +++ b/tests/yamlparser/error_fixtures/unknown_topmost_element_type.error @@ -0,0 +1,3 @@ +unknown_topmost_element_type.yaml:4:3: Unknown topmost element type : 'unknown_element_type'; known are: defaults,job,job-template,job-group,view,view-template,view-group,project,parameter,property,builder,wrapper,trigger,publisher,scm,pipeline-scm,reporter. + - unknown_element_type: + ^ diff --git a/tests/yamlparser/error_fixtures/unknown_topmost_element_type.yaml b/tests/yamlparser/error_fixtures/unknown_topmost_element_type.yaml new file mode 100644 index 000000000..d73f12a4a --- /dev/null +++ b/tests/yamlparser/error_fixtures/unknown_topmost_element_type.yaml @@ -0,0 +1,5 @@ +- job: + name: sample-job + +- unknown_element_type: + name: sample_error diff --git a/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job.error b/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job.error new file mode 100644 index 000000000..eed89f72d --- /dev/null +++ b/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job.error @@ -0,0 +1,7 @@ +xml_generator_builder_error_in_job.yaml:3:3: In job 'sample-job' + - job: + ^ +xml_generator_builder_error_in_job.yaml:6:9: In builder 'shell' + - shell: + ^ +Missing command from an instance of 'builder.shell' diff --git a/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job.yaml b/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job.yaml new file mode 100644 index 000000000..3f19025c9 --- /dev/null +++ b/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job.yaml @@ -0,0 +1,12 @@ +# For XML generator functions context should be added. + +- job: + name: sample-job + builders: + - shell: + wrong_element: hello + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job_template.error b/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job_template.error new file mode 100644 index 000000000..6e5bce881 --- /dev/null +++ b/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job_template.error @@ -0,0 +1,13 @@ +xml_generator_builder_error_in_job_template.yaml:9:3: In project 'sample-project' + - project: + ^ +xml_generator_builder_error_in_job_template.yaml:12:9: Defined here + - sample-job + ^ +xml_generator_builder_error_in_job_template.yaml:3:3: In job template 'sample-job' + - job-template: + ^ +xml_generator_builder_error_in_job_template.yaml:6:9: In builder 'shell' + - shell: + ^ +Missing command from an instance of 'builder.shell' diff --git a/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job_template.yaml b/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job_template.yaml new file mode 100644 index 000000000..ad047ae66 --- /dev/null +++ b/tests/yamlparser/error_fixtures/xml_generator_builder_error_in_job_template.yaml @@ -0,0 +1,12 @@ +# For XML generator functions context should be added. + +- job-template: + name: sample-job + builders: + - shell: + wrong_element: hello + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/test_dimensions.py b/tests/yamlparser/test_dimensions.py index b3d568a52..900d47ada 100644 --- a/tests/yamlparser/test_dimensions.py +++ b/tests/yamlparser/test_dimensions.py @@ -1,6 +1,7 @@ import pytest -from jenkins_jobs.dimensions import DimensionsExpander +from jenkins_jobs.loc_loader import LocDict, LocList +from jenkins_jobs.dimensions import enum_dimensions_params, is_point_included # Axes, params, exclude, expected resulting params. @@ -197,10 +198,9 @@ cases = [ @pytest.mark.parametrize("axes,params,exclude,expected_dimension_params", cases) def test_dimensions(axes, params, exclude, expected_dimension_params): - dim_expander = DimensionsExpander(context=None) dimension_params = [ p - for p in dim_expander.enum_dimensions_params(axes, params, defaults={}) - if dim_expander.is_point_included(exclude, p) + for p in enum_dimensions_params(axes, LocDict(params), defaults={}) + if is_point_included(LocList(exclude), p) ] assert dimension_params == expected_dimension_params diff --git a/tests/yamlparser/test_errors.py b/tests/yamlparser/test_errors.py index 1bbab1dbf..dc8db476c 100644 --- a/tests/yamlparser/test_errors.py +++ b/tests/yamlparser/test_errors.py @@ -21,6 +21,7 @@ from pathlib import Path import pytest +from jenkins_jobs.errors import JenkinsJobsException from tests.enum_scenarios import scenario_list fixtures_dir = Path(__file__).parent / "error_fixtures" @@ -47,6 +48,12 @@ def plugins_info(): def test_error(check_parser, scenario, expected_error): - with pytest.raises(Exception) as excinfo: + with pytest.raises(JenkinsJobsException) as excinfo: check_parser(scenario.in_path) - assert str(excinfo.value) == expected_error + error = "\n".join(excinfo.value.lines) + print() + print(error) + canonical_error = error.replace(str(fixtures_dir) + "/", "").replace( + str(fixtures_dir), "fixtures-dir" + ) + assert canonical_error == expected_error