From 6bc5398336376b245d9133290544a36246ba1720 Mon Sep 17 00:00:00 2001 From: Adam Romanek Date: Tue, 28 Jul 2020 14:37:16 +0200 Subject: [PATCH] Introduce a new '!j2-yaml:' tag The tag provides Jinja templating capabilities for generating parts of YAML structures. Two potential use cases are provided as test cases and are linked in the documentation. The new tag should also help address some use cases people were asking about, like here: https://groups.google.com/g/jenkins-job-builder/c/HkVZVuBDlKM. Change-Id: I96392e42c3c79a9be0a8f736506908701251dd62 --- jenkins_jobs/formatter.py | 7 +- jenkins_jobs/local_yaml.py | 46 ++++++++++++ tests/yamlparser/fixtures/jinja-yaml01.xml | 72 +++++++++++++++++++ tests/yamlparser/fixtures/jinja-yaml01.yaml | 30 ++++++++ tests/yamlparser/fixtures/jinja-yaml02.xml | 48 +++++++++++++ tests/yamlparser/fixtures/jinja-yaml02.yaml | 19 +++++ tests/yamlparser/fixtures/jinja-yaml03.groovy | 3 + tests/yamlparser/fixtures/jinja-yaml03.xml | 27 +++++++ tests/yamlparser/fixtures/jinja-yaml03.yaml | 15 ++++ 9 files changed, 265 insertions(+), 2 deletions(-) create mode 100644 tests/yamlparser/fixtures/jinja-yaml01.xml create mode 100644 tests/yamlparser/fixtures/jinja-yaml01.yaml create mode 100644 tests/yamlparser/fixtures/jinja-yaml02.xml create mode 100644 tests/yamlparser/fixtures/jinja-yaml02.yaml create mode 100644 tests/yamlparser/fixtures/jinja-yaml03.groovy create mode 100644 tests/yamlparser/fixtures/jinja-yaml03.xml create mode 100644 tests/yamlparser/fixtures/jinja-yaml03.yaml diff --git a/jenkins_jobs/formatter.py b/jenkins_jobs/formatter.py index 470646f0e..e47b35c7e 100644 --- a/jenkins_jobs/formatter.py +++ b/jenkins_jobs/formatter.py @@ -79,9 +79,12 @@ def deep_format(obj, paramdict, allow_empty=False): else: ret = obj if isinstance(ret, CustomLoader): - # If we have a CustomLoader here, we've lazily-loaded a template; + # If we have a CustomLoader here, we've lazily-loaded a template + # or rendered a template to a piece of YAML; # attempt to format it. - ret = deep_format(ret, paramdict, allow_empty=allow_empty) + ret = deep_format( + ret.get_object_to_format(), paramdict, allow_empty=allow_empty + ) return ret diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py index 9aeeb9afc..cece2eadb 100644 --- a/jenkins_jobs/local_yaml.py +++ b/jenkins_jobs/local_yaml.py @@ -196,6 +196,24 @@ construct. Examples: .. literalinclude:: /../../tests/yamlparser/fixtures/jinja-string01.yaml + +The tag ``!j2-yaml:`` is similar to the ``!j2:`` tag, just that it loads the +Jinja-rendered string as YAML and embeds it in the calling YAML construct. This +provides a very flexible and convenient way of generating pieces of YAML +structures. One of use cases is defining complex YAML structures with much +simpler configuration, without any duplication. + +Examples: + + .. literalinclude:: /../../tests/yamlparser/fixtures/jinja-yaml01.yaml + +Another use case is controlling lists dynamically, like conditionally adding +list elements based on project configuration. + +Examples: + + .. literalinclude:: /../../tests/yamlparser/fixtures/jinja-yaml02.yaml + """ import functools @@ -369,6 +387,14 @@ class BaseYAMLObject(YAMLObject): yaml_dumper = LocalDumper +class J2Yaml(BaseYAMLObject): + yaml_tag = u"!j2-yaml:" + + @classmethod + def from_yaml(cls, loader, node): + return Jinja2YamlLoader(node.value, loader.search_path) + + class J2String(BaseYAMLObject): yaml_tag = u"!j2:" @@ -572,6 +598,26 @@ class Jinja2Loader(CustomLoader): self._template.environment.loader = self._loader return self._template.render(kwargs) + def get_object_to_format(self): + return self + + +class LateYamlLoader(CustomLoader): + """A loader for data rendered via Jinja2, to be loaded as YAML and then deep formatted.""" + + def __init__(self, yaml_str, loader): + self._yaml_str = yaml_str + self._loader = loader + + def get_object_to_format(self): + return load(self._yaml_str, search_path=self._loader._search_path) + + +class Jinja2YamlLoader(Jinja2Loader): + def format(self, **kwargs): + yaml_str = super(Jinja2YamlLoader, self).format(**kwargs) + return LateYamlLoader(yaml_str, self) + class CustomLoaderCollection(object): """Helper class to format a collection of CustomLoader objects""" diff --git a/tests/yamlparser/fixtures/jinja-yaml01.xml b/tests/yamlparser/fixtures/jinja-yaml01.xml new file mode 100644 index 000000000..ec26a8078 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja-yaml01.xml @@ -0,0 +1,72 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + + + + REG_EXP + a|b|c + + + PLAIN + master + + + false + + + REG_EXP + d|e|f + + + PLAIN + stable + + + false + + + + false + false + false + false + + false + false + true + + BASE64 + PLAIN + PLAIN + BASE64 + + false + + + + + + + + + + + + __ANY__ + + + + + + diff --git a/tests/yamlparser/fixtures/jinja-yaml01.yaml b/tests/yamlparser/fixtures/jinja-yaml01.yaml new file mode 100644 index 000000000..252c93a5f --- /dev/null +++ b/tests/yamlparser/fixtures/jinja-yaml01.yaml @@ -0,0 +1,30 @@ +- job-template: + name: test-job-template + triggers: + - gerrit: + projects: + !j2-yaml: | + {% for item in triggers %} + - branches: + - branch-compare-type: PLAIN + branch-pattern: '{{ item.branch }}' + project-compare-type: REG_EXP + project-pattern: '{{ item.repositories|join("|") }}' + {% endfor %} + +- project: + name: test-job-project + + jobs: + - test-job-template: + triggers: + - repositories: + - a + - b + - c + branch: master + - repositories: + - d + - e + - f + branch: stable diff --git a/tests/yamlparser/fixtures/jinja-yaml02.xml b/tests/yamlparser/fixtures/jinja-yaml02.xml new file mode 100644 index 000000000..4c6f687a9 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja-yaml02.xml @@ -0,0 +1,48 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + false + false + + + + + + + + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + false + false + + + + 7 + -1 + -1 + -1 + + + + + + + + diff --git a/tests/yamlparser/fixtures/jinja-yaml02.yaml b/tests/yamlparser/fixtures/jinja-yaml02.yaml new file mode 100644 index 000000000..7477279c5 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja-yaml02.yaml @@ -0,0 +1,19 @@ +- job-template: + name: 'test-job-{variant}' + properties: !j2-yaml: | + - rebuild + {% if discard_old_builds|default %} + - build-discarder: + days-to-keep: 7 + {% endif %} + +- project: + name: test-project + + jobs: + - 'test-job-{variant}': + variant: abc + + - 'test-job-{variant}': + variant: def + discard_old_builds: true diff --git a/tests/yamlparser/fixtures/jinja-yaml03.groovy b/tests/yamlparser/fixtures/jinja-yaml03.groovy new file mode 100644 index 000000000..bc5b62c94 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja-yaml03.groovy @@ -0,0 +1,3 @@ +if (manager.logContains(".*no_jenkins.*")) { + manager.build.result = hudson.model.Result.NOT_BUILT +} diff --git a/tests/yamlparser/fixtures/jinja-yaml03.xml b/tests/yamlparser/fixtures/jinja-yaml03.xml new file mode 100644 index 000000000..052f28641 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja-yaml03.xml @@ -0,0 +1,27 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + + 0 + false + + false + + + + + diff --git a/tests/yamlparser/fixtures/jinja-yaml03.yaml b/tests/yamlparser/fixtures/jinja-yaml03.yaml new file mode 100644 index 000000000..06a32b5be --- /dev/null +++ b/tests/yamlparser/fixtures/jinja-yaml03.yaml @@ -0,0 +1,15 @@ +# the purpose of this test is to check if the piece of YAML generated by +# !j2-yaml is deep-formatted properly; if not then double quotes introduced by +# !include-raw-escape would be left untouched and passed down to the output XML +# file, which would simply be wrong... + +- job-template: + name: 'test-job-template' + publishers: !j2-yaml: | + - groovy-postbuild: + script: !include-raw-escape: ./jinja-yaml03.groovy + +- project: + name: test-project + jobs: + - 'test-job-template'