471 lines
16 KiB
Python
471 lines
16 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# Provides local yaml parsing classes and extends yaml module.
|
|
|
|
"""Custom application specific yamls tags are supported to provide
|
|
enhancements when reading yaml configuration.
|
|
|
|
Action Tags
|
|
^^^^^^^^^^^
|
|
|
|
These allow manipulation of data being stored in one layout in the source
|
|
yaml for convenience and/or clarity, to another format to be processed by
|
|
the targeted module instead of requiring all modules in JJB being capable
|
|
of supporting multiple input formats.
|
|
|
|
The tag ``!join:`` will treat the first element of the following list as
|
|
the delimiter to use, when joining the remaining elements into a string
|
|
and returning a single string to be consumed by the specified module option.
|
|
|
|
This allows users to maintain elements of data in a list structure for ease
|
|
of review/maintenance, and have the yaml parser convert it to a string for
|
|
consumption as any argument for modules. The main expected use case is to
|
|
allow for generic plugin data such as shell properties to be populated from
|
|
a list construct which the yaml parser converts to a single string, instead
|
|
of trying to support this within the module code which would require a
|
|
templating engine similar to Jinja.
|
|
|
|
Generic Example:
|
|
|
|
.. literalinclude:: /../../tests/loader/fixtures/joinlists.yaml
|
|
|
|
|
|
Environment Inject:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/string_join.yaml
|
|
|
|
|
|
While this mechanism can also be used items where delimiters are supported by
|
|
the module, that should be considered a bug that the existing code doesn't
|
|
handle being provided a list and delimiter to perform the correct conversion
|
|
for you. Should you discover a module that takes arguments with delimiters and
|
|
the existing JJB codebase does not handle accepting lists, then this can be
|
|
used as a temporary solution in place of using very long strings:
|
|
|
|
Extended Params Example:
|
|
|
|
.. literalinclude::
|
|
/../../tests/parameters/fixtures/extended-choice-param-full.yaml
|
|
|
|
|
|
Inclusion Tags
|
|
^^^^^^^^^^^^^^
|
|
|
|
These allow inclusion of arbitrary files as a method of having blocks of data
|
|
managed separately to the yaml job configurations. A specific usage of this is
|
|
inlining scripts contained in separate files, although such tags may also be
|
|
used to simplify usage of macros or job templates.
|
|
|
|
The tag ``!include:`` will treat the following string as file which should be
|
|
parsed as yaml configuration data.
|
|
|
|
Example:
|
|
|
|
.. literalinclude:: /../../tests/loader/fixtures/include001.yaml
|
|
|
|
contents of include001.yaml.inc:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/include001.yaml.inc
|
|
|
|
|
|
The tag ``!include-raw:`` will treat the given string or list of strings as
|
|
filenames to be opened as one or more data blob, which should be read into
|
|
the calling yaml construct without any further parsing. Any data in a file
|
|
included through this tag, will be treated as string data.
|
|
|
|
Examples:
|
|
|
|
.. literalinclude:: /../../tests/loader/fixtures/include-raw001-job.yaml
|
|
|
|
contents of include-raw001-hello-world.sh:
|
|
|
|
.. literalinclude::
|
|
/../../tests/loader/fixtures/include-raw001-hello-world.sh
|
|
|
|
contents of include-raw001-vars.sh:
|
|
|
|
.. literalinclude::
|
|
/../../tests/loader/fixtures/include-raw001-vars.sh
|
|
|
|
using a list of files:
|
|
|
|
.. literalinclude::
|
|
/../../tests/loader/fixtures/include-raw-multi001.yaml
|
|
|
|
The tag ``!include-raw-escape:`` treats the given string or list of strings as
|
|
filenames to be opened as one or more data blobs, which should be escaped
|
|
before being read in as string data. This allows job-templates to use this tag
|
|
to include scripts from files without needing to escape braces in the original
|
|
file.
|
|
|
|
.. warning::
|
|
|
|
When used as a macro ``!include-raw-escape:`` should only be used if
|
|
parameters are passed into the escaped file and you would like to escape
|
|
those parameters. If the file does not have any jjb parameters passed into
|
|
it then ``!include-raw:`` should be used instead otherwise you will run
|
|
into an interesting issue where ``include-raw-escape:`` actually adds
|
|
additional curly braces around existing curly braces. For example
|
|
${PROJECT} becomes ${{PROJECT}} which may break bash scripts.
|
|
|
|
Examples:
|
|
|
|
.. literalinclude::
|
|
/../../tests/loader/fixtures/include-raw-escaped001-template.yaml
|
|
|
|
contents of include-raw001-hello-world.sh:
|
|
|
|
.. literalinclude::
|
|
/../../tests/loader/fixtures/include-raw001-hello-world.sh
|
|
|
|
contents of include-raw001-vars.sh:
|
|
|
|
.. literalinclude::
|
|
/../../tests/loader/fixtures/include-raw001-vars.sh
|
|
|
|
using a list of files:
|
|
|
|
.. literalinclude::
|
|
/../../tests/loader/fixtures/include-raw-escaped-multi001.yaml
|
|
|
|
|
|
For all the multi file includes, the files are simply appended using a newline
|
|
character.
|
|
|
|
|
|
To allow for job templates to perform substitution on the path names, when a
|
|
filename containing a python format placeholder is encountered, lazy loading
|
|
support is enabled, where instead of returning the contents back during yaml
|
|
parsing, it is delayed until the variable substitution is performed.
|
|
|
|
Example:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/lazy-load-jobs001.yaml
|
|
|
|
using a list of files:
|
|
|
|
.. literalinclude::
|
|
/../../tests/yamlparser/job_fixtures/lazy-load-jobs-multi001.yaml
|
|
|
|
.. note::
|
|
|
|
Because lazy-loading involves performing the substitution on the file
|
|
name, it means that jenkins-job-builder can not call the variable
|
|
substitution on the contents of the file. This means that the
|
|
``!include-raw:`` tag will behave as though ``!include-raw-escape:`` tag
|
|
was used instead whenever name substitution on the filename is to be
|
|
performed.
|
|
|
|
Given the behaviour described above, when substitution is to be performed
|
|
on any filename passed via ``!include-raw-escape:`` the tag will be
|
|
automatically converted to ``!include-raw:`` and no escaping will be
|
|
performed.
|
|
|
|
|
|
The tag ``!include-jinja2:`` will treat the given string or list of strings as
|
|
filenames to be opened as Jinja2 templates, which should be rendered to a
|
|
string and included in the calling YAML construct. (This is analogous to the
|
|
templating that will happen with ``!include-raw``.)
|
|
|
|
Examples:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja01.yaml
|
|
|
|
contents of jinja01.yaml.inc:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja01.yaml.inc
|
|
|
|
|
|
The tag ``!j2:`` takes a string and treats it as a Jinja2 template. It will be
|
|
rendered (with the variables in that context) and included in the calling YAML
|
|
construct.
|
|
|
|
Examples:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja-string01.yaml
|
|
|
|
The tag ``!j2-yaml:`` is similar to the ``!j2:`` tag, just that it loads the
|
|
Jinja-rendered string as YAML and embeds it in the calling YAML construct. This
|
|
provides a very flexible and convenient way of generating pieces of YAML
|
|
structures. One of use cases is defining complex YAML structures with much
|
|
simpler configuration, without any duplication.
|
|
|
|
Examples:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja-yaml01.yaml
|
|
|
|
Another use case is controlling lists dynamically, like conditionally adding
|
|
list elements based on project configuration.
|
|
|
|
Examples:
|
|
|
|
.. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja-yaml02.yaml
|
|
|
|
"""
|
|
|
|
import abc
|
|
import os.path
|
|
import logging
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import jinja2
|
|
import jinja2.meta
|
|
import yaml
|
|
|
|
from .errors import JenkinsJobsException
|
|
from .formatter import CustomFormatter, enum_str_format_required_params
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
if sys.version_info >= (3, 8):
|
|
from functools import cached_property
|
|
else:
|
|
from functools import lru_cache
|
|
|
|
# cached_property was introduced in python 3.8.
|
|
# Recipe from https://stackoverflow.com/a/19979379
|
|
def cached_property(fn):
|
|
return property(lru_cache()(fn))
|
|
|
|
|
|
class BaseYamlObject(metaclass=abc.ABCMeta):
|
|
@staticmethod
|
|
def path_list_from_node(loader, node):
|
|
if isinstance(node, yaml.ScalarNode):
|
|
return [loader.construct_yaml_str(node)]
|
|
elif isinstance(node, yaml.SequenceNode):
|
|
return loader.construct_sequence(node)
|
|
else:
|
|
raise yaml.constructor.ConstructorError(
|
|
None,
|
|
None,
|
|
f"expected either a sequence or scalar node, but found {node.id}",
|
|
node.start_mark,
|
|
)
|
|
|
|
@classmethod
|
|
def from_yaml(cls, loader, node):
|
|
value = loader.construct_yaml_str(node)
|
|
return cls(loader.jjb_config, loader, value)
|
|
|
|
def __init__(self, jjb_config, loader):
|
|
self._search_path = jjb_config.yamlparser["include_path"]
|
|
if loader.source_path:
|
|
# Loaded from a file, find includes beside it too.
|
|
self._search_path.append(os.path.dirname(loader.source_path))
|
|
self._loader = loader
|
|
allow_empty = jjb_config.yamlparser["allow_empty_variables"]
|
|
self._formatter = CustomFormatter(allow_empty)
|
|
|
|
@abc.abstractmethod
|
|
def expand(self, expander, params):
|
|
"""Expand object but do not substitute template parameters"""
|
|
pass
|
|
|
|
def subst(self, expander, params):
|
|
"""Expand object and substitute template parameters"""
|
|
return self.expand(expander, params)
|
|
|
|
def _find_file(self, rel_path):
|
|
search_path = self._search_path
|
|
if "." not in search_path:
|
|
search_path.append(".")
|
|
dir_list = [Path(d).expanduser() for d in self._search_path]
|
|
for dir in dir_list:
|
|
candidate = dir.joinpath(rel_path)
|
|
if candidate.is_file():
|
|
logger.debug("Including file %r from path %r", str(rel_path), str(dir))
|
|
return candidate
|
|
raise JenkinsJobsException(
|
|
f"File {rel_path} does not exist on any of include directories:"
|
|
f" {','.join([str(d) for d in dir_list])}"
|
|
)
|
|
|
|
|
|
class J2BaseYamlObject(BaseYamlObject):
|
|
def __init__(self, jjb_config, loader):
|
|
super().__init__(jjb_config, loader)
|
|
self._jinja2_env = jinja2.Environment(
|
|
loader=jinja2.FileSystemLoader(self._search_path),
|
|
undefined=jinja2.StrictUndefined,
|
|
)
|
|
|
|
@staticmethod
|
|
def _render_template(template_text, template, params):
|
|
try:
|
|
return template.render(params)
|
|
except jinja2.UndefinedError as x:
|
|
if len(template_text) > 40:
|
|
text = template_text[:40] + "..."
|
|
else:
|
|
text = template_text
|
|
raise JenkinsJobsException(
|
|
f"While formatting jinja2 template {text!r}: {x}"
|
|
)
|
|
|
|
|
|
class J2Template(J2BaseYamlObject):
|
|
def __init__(self, jjb_config, loader, template_text):
|
|
super().__init__(jjb_config, loader)
|
|
self._template_text = template_text
|
|
self._template = self._jinja2_env.from_string(template_text)
|
|
|
|
@cached_property
|
|
def required_params(self):
|
|
ast = self._jinja2_env.parse(self._template_text)
|
|
return jinja2.meta.find_undeclared_variables(ast)
|
|
|
|
def _render(self, params):
|
|
return self._render_template(self._template_text, self._template, params)
|
|
|
|
|
|
class J2String(J2Template):
|
|
yaml_tag = "!j2:"
|
|
|
|
def expand(self, expander, params):
|
|
return self._render(params)
|
|
|
|
|
|
class J2Yaml(J2Template):
|
|
yaml_tag = "!j2-yaml:"
|
|
|
|
def expand(self, expander, params):
|
|
text = self._render(params)
|
|
data = self._loader.load(text)
|
|
return expander.expand(data, params)
|
|
|
|
|
|
class IncludeJinja2(J2BaseYamlObject):
|
|
yaml_tag = "!include-jinja2:"
|
|
|
|
@classmethod
|
|
def from_yaml(cls, loader, node):
|
|
path_list = cls.path_list_from_node(loader, node)
|
|
return cls(loader.jjb_config, loader, path_list)
|
|
|
|
def __init__(self, jjb_config, loader, path_list):
|
|
super().__init__(jjb_config, loader)
|
|
self._path_list = path_list
|
|
|
|
@property
|
|
def required_params(self):
|
|
return []
|
|
|
|
def expand(self, expander, params):
|
|
return "\n".join(
|
|
self._expand_path(expander, params, path) for path in self._path_list
|
|
)
|
|
|
|
def _expand_path(self, expander, params, path_template):
|
|
rel_path = self._formatter.format(path_template, **params)
|
|
full_path = self._find_file(rel_path)
|
|
template_text = full_path.read_text()
|
|
template = self._jinja2_env.from_string(template_text)
|
|
return self._render_template(template_text, template, params)
|
|
|
|
|
|
class IncludeBaseObject(BaseYamlObject):
|
|
@classmethod
|
|
def from_yaml(cls, loader, node):
|
|
path_list = cls.path_list_from_node(loader, node)
|
|
return cls(loader.jjb_config, loader, path_list)
|
|
|
|
def __init__(self, jjb_config, loader, path_list):
|
|
super().__init__(jjb_config, loader)
|
|
self._path_list = path_list
|
|
|
|
@property
|
|
def required_params(self):
|
|
for path in self._path_list:
|
|
yield from enum_str_format_required_params(path)
|
|
|
|
|
|
class YamlInclude(IncludeBaseObject):
|
|
yaml_tag = "!include:"
|
|
|
|
def expand(self, expander, params):
|
|
yaml_list = [
|
|
self._expand_path(expander, params, path) for path in self._path_list
|
|
]
|
|
if len(yaml_list) == 1:
|
|
return yaml_list[0]
|
|
else:
|
|
return "\n".join(yaml_list)
|
|
|
|
def _expand_path(self, expander, params, path_template):
|
|
rel_path = self._formatter.format(path_template, **params)
|
|
full_path = self._find_file(rel_path)
|
|
text = full_path.read_text()
|
|
data = self._loader.load(text)
|
|
return expander.expand(data, params)
|
|
|
|
|
|
class IncludeRawBase(IncludeBaseObject):
|
|
def expand(self, expander, params):
|
|
return "\n".join(self._expand_path(path, params) for path in self._path_list)
|
|
|
|
def subst(self, expander, params):
|
|
return "\n".join(self._subst_path(path, params) for path in self._path_list)
|
|
|
|
|
|
class IncludeRaw(IncludeRawBase):
|
|
yaml_tag = "!include-raw:"
|
|
|
|
def _expand_path(self, rel_path_template, params):
|
|
rel_path = self._formatter.format(rel_path_template, **params)
|
|
full_path = self._find_file(rel_path)
|
|
return full_path.read_text()
|
|
|
|
def _subst_path(self, rel_path_template, params):
|
|
rel_path = self._formatter.format(rel_path_template, **params)
|
|
full_path = self._find_file(rel_path)
|
|
template = full_path.read_text()
|
|
return self._formatter.format(template, **params)
|
|
|
|
|
|
class IncludeRawEscape(IncludeRawBase):
|
|
yaml_tag = "!include-raw-escape:"
|
|
|
|
def _expand_path(self, rel_path_template, params):
|
|
rel_path = self._formatter.format(rel_path_template, **params)
|
|
full_path = self._find_file(rel_path)
|
|
text = full_path.read_text()
|
|
# Backward compatibility:
|
|
# if used inside job or macro without parameters, curly braces are duplicated.
|
|
return text.replace("{", "{{").replace("}", "}}")
|
|
|
|
def _subst_path(self, rel_path_template, params):
|
|
rel_path = self._formatter.format(rel_path_template, **params)
|
|
full_path = self._find_file(rel_path)
|
|
return full_path.read_text()
|
|
|
|
|
|
class YamlListJoin:
|
|
yaml_tag = "!join:"
|
|
|
|
@classmethod
|
|
def from_yaml(cls, loader, node):
|
|
value = loader.construct_sequence(node, deep=True)
|
|
if len(value) != 2:
|
|
raise yaml.constructor.ConstructorError(
|
|
None,
|
|
None,
|
|
"Join value should contain 2 elements: delimiter and string list,"
|
|
f" but contains {len(value)} elements: {value!r}",
|
|
node.start_mark,
|
|
)
|
|
delimiter, seq = value
|
|
return delimiter.join(seq)
|