18efe5066b
Expand variables inside macros without parameters and jobs the same way as they are expanded inside macros with parameters and job templates. Make tags behave inside macros without parameters and jobs the same way as they are expanded inside macros with parameters and job templates. Update or fix affected tests. Story: 2010588 Story: 2010963 Story: 2010535 Task: 47394 Task: 49069 Task: 47151 Change-Id: Ie05ae6aa386c62ebbf68dd3e2c7001a4e444a47a
439 lines
15 KiB
Python
439 lines
15 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# Provides local yaml parsing classes and extends yaml module.
|
|
|
|
"""Custom application specific yamls tags are supported to provide
|
|
enhancements when reading yaml configuration.
|
|
|
|
Action Tags
|
|
^^^^^^^^^^^
|
|
|
|
These allow manipulation of data being stored in one layout in the source
|
|
yaml for convenience and/or clarity, to another format to be processed by
|
|
the targeted module instead of requiring all modules in JJB being capable
|
|
of supporting multiple input formats.
|
|
|
|
The tag ``!join:`` will treat the first element of the following list as
|
|
the delimiter to use, when joining the remaining elements into a string
|
|
and returning a single string to be consumed by the specified module option.
|
|
|
|
This allows users to maintain elements of data in a list structure for ease
|
|
of review/maintenance, and have the yaml parser convert it to a string for
|
|
consumption as any argument for modules. The main expected use case is to
|
|
allow for generic plugin data such as shell properties to be populated from
|
|
a list construct which the yaml parser converts to a single string, instead
|
|
of trying to support this within the module code which would require a
|
|
templating engine similar to Jinja.
|
|
|
|
Generic Example:
|
|
|
|
.. literalinclude:: /../../tests/loader/fixtures/joinlists.yaml
|
|
|
|
|
|
Environment Inject:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/string_join.yaml
|
|
|
|
|
|
While this mechanism can also be used items where delimiters are supported by
|
|
the module, that should be considered a bug that the existing code doesn't
|
|
handle being provided a list and delimiter to perform the correct conversion
|
|
for you. Should you discover a module that takes arguments with delimiters and
|
|
the existing JJB codebase does not handle accepting lists, then this can be
|
|
used as a temporary solution in place of using very long strings:
|
|
|
|
Extended Params Example:
|
|
|
|
.. literalinclude::
|
|
/../../tests/parameters/fixtures/extended-choice-param-full.yaml
|
|
|
|
|
|
Inclusion Tags
|
|
^^^^^^^^^^^^^^
|
|
|
|
These allow inclusion of arbitrary files as a method of having blocks of data
|
|
managed separately to the yaml job configurations. A specific usage of this is
|
|
inlining scripts contained in separate files, although such tags may also be
|
|
used to simplify usage of macros or job templates.
|
|
|
|
The tag ``!include:`` will treat the following string as file which should be
|
|
parsed as yaml configuration data.
|
|
|
|
Example:
|
|
|
|
.. literalinclude:: /../../tests/loader/fixtures/include001.yaml
|
|
|
|
contents of include001.yaml.inc:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/include001.yaml.inc
|
|
|
|
|
|
The tag ``!include-raw:`` will treat the given string or list of strings as
|
|
filenames to be opened as one or more data blob, which should be read into
|
|
the calling yaml construct without any further parsing. Any data in a file
|
|
included through this tag, will be treated as string data.
|
|
|
|
It will expand variables inside the file. If your file contains curly braces,
|
|
you should double them. Or, you can use tag ``!include-raw-escape``, which
|
|
does not substitute variables.
|
|
|
|
Examples:
|
|
|
|
.. literalinclude::
|
|
/../../tests/loader/fixtures/include-raw-escaped001-template.yaml
|
|
|
|
contents of include-raw001-hello-world.sh:
|
|
|
|
.. literalinclude::
|
|
/../../tests/loader/fixtures/include-raw001-hello-world.sh
|
|
|
|
contents of include-raw001-vars.sh:
|
|
|
|
.. literalinclude::
|
|
/../../tests/loader/fixtures/include-raw001-vars.sh
|
|
|
|
using a list of files:
|
|
|
|
.. literalinclude::
|
|
/../../tests/loader/fixtures/include-raw-escaped-multi001.yaml
|
|
|
|
|
|
For all the multi file includes, the files are simply appended using a newline
|
|
character.
|
|
|
|
|
|
You can also use variables in included file paths.
|
|
|
|
Example:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/lazy-load-jobs001.yaml
|
|
|
|
with variable substitution inside included files:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/lazy-load-with-variables.yaml
|
|
|
|
using a list of files:
|
|
|
|
.. literalinclude::
|
|
/../../tests/yamlparser/job_fixtures/lazy-load-jobs-multi001.yaml
|
|
|
|
|
|
The tag ``!include-jinja2:`` will treat the given string or list of strings as
|
|
filenames to be opened as Jinja2 templates, which should be rendered to a
|
|
string and included in the calling YAML construct. (This is analogous to the
|
|
templating that will happen with ``!include-raw``.)
|
|
|
|
Examples:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja01.yaml
|
|
|
|
contents of jinja01.yaml.inc:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja01.yaml.inc
|
|
|
|
|
|
The tag ``!j2:`` takes a string and treats it as a Jinja2 template. It will be
|
|
rendered (with the variables in that context) and included in the calling YAML
|
|
construct.
|
|
|
|
Examples:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja-string01.yaml
|
|
|
|
The tag ``!j2-yaml:`` is similar to the ``!j2:`` tag, just that it loads the
|
|
Jinja-rendered string as YAML and embeds it in the calling YAML construct. This
|
|
provides a very flexible and convenient way of generating pieces of YAML
|
|
structures. One of use cases is defining complex YAML structures with much
|
|
simpler configuration, without any duplication.
|
|
|
|
Examples:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja-yaml01.yaml
|
|
|
|
Another use case is controlling lists dynamically, like conditionally adding
|
|
list elements based on project configuration.
|
|
|
|
Examples:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja-yaml02.yaml
|
|
|
|
"""
|
|
|
|
import abc
|
|
import logging
|
|
import traceback
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import jinja2
|
|
import jinja2.meta
|
|
import yaml
|
|
|
|
from .errors import Context, JenkinsJobsException
|
|
from .loc_loader import LocList
|
|
from .position import Pos
|
|
from .formatter import CustomFormatter, enum_str_format_required_params
|
|
|
|
if sys.version_info >= (3, 8):
|
|
from functools import cached_property
|
|
else:
|
|
from .cached_property import cached_property
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class BaseYamlObject(metaclass=abc.ABCMeta):
|
|
@staticmethod
|
|
def path_list_from_node(loader, node):
|
|
if isinstance(node, yaml.ScalarNode):
|
|
return LocList(
|
|
[loader.construct_yaml_str(node)],
|
|
value_pos=[loader.pos_from_node(node)],
|
|
)
|
|
elif isinstance(node, yaml.SequenceNode):
|
|
return LocList(
|
|
loader.construct_sequence(node),
|
|
value_pos=[loader.pos_from_node(n) for n in node.value],
|
|
)
|
|
else:
|
|
raise JenkinsJobsException(
|
|
f"Expected either a sequence or scalar node, but found {node.id}",
|
|
pos=loader.pos_from_node(node),
|
|
)
|
|
|
|
@classmethod
|
|
def from_yaml(cls, loader, node):
|
|
value = loader.construct_yaml_str(node)
|
|
return cls(loader.jjb_config, loader, loader.pos_from_node(node), value)
|
|
|
|
def __init__(self, jjb_config, loader, pos):
|
|
self._search_path = jjb_config.yamlparser["include_path"].copy()
|
|
if loader.source_dir:
|
|
# Loaded from a file, find includes beside it too.
|
|
self._search_path.append(loader.source_dir)
|
|
self._loader = loader
|
|
self._pos = pos
|
|
allow_empty = jjb_config.yamlparser["allow_empty_variables"]
|
|
self._formatter = CustomFormatter(allow_empty)
|
|
|
|
@abc.abstractmethod
|
|
def expand(self, expander, params):
|
|
"""Expand object and substitute template parameters"""
|
|
pass
|
|
|
|
def _find_file(self, rel_path, pos):
|
|
search_path = self._search_path
|
|
if "." not in search_path:
|
|
search_path.append(".")
|
|
dir_list = [Path(d).expanduser() for d in self._search_path]
|
|
for dir in dir_list:
|
|
candidate = dir.joinpath(rel_path)
|
|
if candidate.is_file():
|
|
logger.debug("Including file %r from path %r", str(rel_path), str(dir))
|
|
return candidate
|
|
dir_list_str = ",".join(str(d) for d in dir_list)
|
|
raise JenkinsJobsException(
|
|
f"File {rel_path} does not exist in any of include directories: {dir_list_str}",
|
|
pos=pos,
|
|
)
|
|
|
|
def _expand_path_list(self, path_list, *args):
|
|
for idx, path in enumerate(path_list):
|
|
yield self._expand_path(path, path_list.value_pos[idx], *args)
|
|
|
|
|
|
class J2BaseYamlObject(BaseYamlObject):
|
|
def __init__(self, jjb_config, loader, pos):
|
|
super().__init__(jjb_config, loader, pos)
|
|
self._jinja2_env = jinja2.Environment(
|
|
loader=jinja2.FileSystemLoader(self._search_path),
|
|
undefined=jinja2.StrictUndefined,
|
|
)
|
|
|
|
def _render_template(self, pos, template_text, template, params):
|
|
try:
|
|
return template.render(params)
|
|
except jinja2.UndefinedError as x:
|
|
# Jinja2 adds fake traceback entry with template line number.
|
|
tb = traceback.extract_tb(x.__traceback__)
|
|
line_ofs = tb[-1].lineno - 1 # traceback lineno starts with 1.
|
|
lines = template_text.splitlines()
|
|
start_ofs = pos.body.index(lines[0])
|
|
# Examples for pre_pad: '!j2: \n<indent spaces>', '!j2: '.
|
|
pre_pad = pos.body[:start_ofs]
|
|
# Shift position to reflect template position inside yaml file:
|
|
if "\n" in pre_pad:
|
|
pos = pos.with_offset(line_ofs=1)
|
|
column_ofs = 0
|
|
else:
|
|
column_ofs = start_ofs
|
|
# Move position to error inside template:
|
|
pos = pos.with_offset(line_ofs, column_ofs)
|
|
pos = pos.with_contents_start()
|
|
if len(template_text) > 40:
|
|
text = template_text[:40] + "..."
|
|
else:
|
|
text = template_text
|
|
context = Context(f"While formatting jinja2 template {text!r}", self._pos)
|
|
raise JenkinsJobsException(str(x), pos=pos, ctx=[context])
|
|
|
|
|
|
class J2Template(J2BaseYamlObject):
|
|
def __init__(self, jjb_config, loader, pos, template_text):
|
|
super().__init__(jjb_config, loader, pos)
|
|
self._template_text = template_text
|
|
self._template = self._jinja2_env.from_string(template_text)
|
|
|
|
@cached_property
|
|
def required_params(self):
|
|
ast = self._jinja2_env.parse(self._template_text)
|
|
return jinja2.meta.find_undeclared_variables(ast)
|
|
|
|
def _render(self, params):
|
|
return self._render_template(
|
|
self._pos, self._template_text, self._template, params
|
|
)
|
|
|
|
|
|
class J2String(J2Template):
|
|
yaml_tag = "!j2:"
|
|
|
|
def expand(self, expander, params):
|
|
return self._render(params)
|
|
|
|
|
|
class J2Yaml(J2Template):
|
|
yaml_tag = "!j2-yaml:"
|
|
|
|
def expand(self, expander, params):
|
|
text = self._render(params)
|
|
data = self._loader.load(
|
|
text, source_path="<expanded j2-yaml>", source_dir=self._loader.source_dir
|
|
)
|
|
try:
|
|
return expander.expand(data, params)
|
|
except JenkinsJobsException as x:
|
|
raise x.with_context("In expanded !j2-yaml:", self._pos)
|
|
|
|
|
|
class IncludeJinja2(J2BaseYamlObject):
|
|
yaml_tag = "!include-jinja2:"
|
|
|
|
@classmethod
|
|
def from_yaml(cls, loader, node):
|
|
path_list = cls.path_list_from_node(loader, node)
|
|
return cls(loader.jjb_config, loader, loader.pos_from_node(node), path_list)
|
|
|
|
def __init__(self, jjb_config, loader, pos, path_list):
|
|
super().__init__(jjb_config, loader, pos)
|
|
self._path_list = path_list
|
|
|
|
@property
|
|
def required_params(self):
|
|
return []
|
|
|
|
def expand(self, expander, params):
|
|
return "\n".join(self._expand_path_list(self._path_list, expander, params))
|
|
|
|
def _expand_path(self, path_template, pos, expander, params):
|
|
rel_path = self._formatter.format(path_template, **params)
|
|
full_path = self._find_file(rel_path, pos)
|
|
template_text = full_path.read_text()
|
|
template = self._jinja2_env.from_string(template_text)
|
|
pos = Pos.from_file(full_path, template_text)
|
|
try:
|
|
return self._render_template(pos, template_text, template, params)
|
|
except JenkinsJobsException as x:
|
|
raise x.with_context(f"In included file {str(full_path)!r}", pos=self._pos)
|
|
|
|
|
|
class IncludeBaseObject(BaseYamlObject):
|
|
@classmethod
|
|
def from_yaml(cls, loader, node):
|
|
path_list = cls.path_list_from_node(loader, node)
|
|
return cls(loader.jjb_config, loader, loader.pos_from_node(node), path_list)
|
|
|
|
def __init__(self, jjb_config, loader, pos, path_list):
|
|
super().__init__(jjb_config, loader, pos)
|
|
self._path_list = path_list
|
|
|
|
@property
|
|
def required_params(self):
|
|
for idx, path in enumerate(self._path_list):
|
|
yield from enum_str_format_required_params(
|
|
path, pos=self._path_list.value_pos[idx]
|
|
)
|
|
|
|
|
|
class YamlInclude(IncludeBaseObject):
|
|
yaml_tag = "!include:"
|
|
|
|
def expand(self, expander, params):
|
|
yaml_list = list(self._expand_path_list(self._path_list, expander, params))
|
|
if len(yaml_list) == 1:
|
|
return yaml_list[0]
|
|
else:
|
|
return "\n".join(yaml_list)
|
|
|
|
def _expand_path(self, path_template, pos, expander, params):
|
|
rel_path = self._formatter.format(path_template, **params)
|
|
full_path = self._find_file(rel_path, pos)
|
|
data = self._loader.load_path(full_path)
|
|
try:
|
|
return expander.expand(data, params)
|
|
except JenkinsJobsException as x:
|
|
raise x.with_context(f"In included file {str(full_path)!r}", pos=self._pos)
|
|
|
|
|
|
class IncludeRawBase(IncludeBaseObject):
|
|
def expand(self, expander, params):
|
|
return "\n".join(self._expand_path_list(self._path_list, params))
|
|
|
|
|
|
class IncludeRaw(IncludeRawBase):
|
|
yaml_tag = "!include-raw:"
|
|
|
|
def _expand_path(self, rel_path_template, pos, params):
|
|
rel_path = self._formatter.format(rel_path_template, **params)
|
|
full_path = self._find_file(rel_path, pos)
|
|
template = full_path.read_text()
|
|
try:
|
|
return self._formatter.format(template, **params)
|
|
except JenkinsJobsException as x:
|
|
raise x.with_context(f"In included file {str(full_path)!r}", pos=self._pos)
|
|
|
|
|
|
class IncludeRawEscape(IncludeRawBase):
|
|
yaml_tag = "!include-raw-escape:"
|
|
|
|
def _expand_path(self, rel_path_template, pos, params):
|
|
rel_path = self._formatter.format(rel_path_template, **params)
|
|
full_path = self._find_file(rel_path, pos)
|
|
return full_path.read_text()
|
|
|
|
|
|
class YamlListJoin:
|
|
yaml_tag = "!join:"
|
|
|
|
@classmethod
|
|
def from_yaml(cls, loader, node):
|
|
value = loader.construct_sequence(node, deep=True)
|
|
if len(value) != 2:
|
|
raise JenkinsJobsException(
|
|
"Join value should contain 2 elements: delimiter and string list,"
|
|
f" but contains {len(value)} elements: {value!r}",
|
|
pos=loader.pos_from_node(node),
|
|
)
|
|
delimiter, seq = value
|
|
return delimiter.join(seq)
|