Merge "Introduce a new '!j2-yaml:' tag"

jenkins_jobs/formatter.py

@@ -82,9 +82,12 @@ def deep_format(obj, paramdict, allow_empty=False):
     else:
         ret = obj
     if isinstance(ret, CustomLoader):
-        # If we have a CustomLoader here, we've lazily-loaded a template;
+        # If we have a CustomLoader here, we've lazily-loaded a template
+        # or rendered a template to a piece of YAML;
         # attempt to format it.
-        ret = deep_format(ret, paramdict, allow_empty=allow_empty)
+        ret = deep_format(
+            ret.get_object_to_format(), paramdict, allow_empty=allow_empty
+        )
     return ret
 
 

jenkins_jobs/local_yaml.py

@@ -196,6 +196,24 @@ construct.
 Examples:
 
     .. literalinclude:: /../../tests/yamlparser/fixtures/jinja-string01.yaml
+
+The tag ``!j2-yaml:`` is similar to the ``!j2:`` tag, just that it loads the
+Jinja-rendered string as YAML and embeds it in the calling YAML construct. This
+provides a very flexible and convenient way of generating pieces of YAML
+structures. One of use cases is defining complex YAML structures with much
+simpler configuration, without any duplication.
+
+Examples:
+
+    .. literalinclude:: /../../tests/yamlparser/fixtures/jinja-yaml01.yaml
+
+Another use case is controlling lists dynamically, like conditionally adding
+list elements based on project configuration.
+
+Examples:
+
+    .. literalinclude:: /../../tests/yamlparser/fixtures/jinja-yaml02.yaml
+
 """
 
 import functools
@@ -369,6 +387,14 @@ class BaseYAMLObject(YAMLObject):
     yaml_dumper = LocalDumper
 
 
+class J2Yaml(BaseYAMLObject):
+    yaml_tag = u"!j2-yaml:"
+
+    @classmethod
+    def from_yaml(cls, loader, node):
+        return Jinja2YamlLoader(node.value, loader.search_path)
+
+
 class J2String(BaseYAMLObject):
     yaml_tag = u"!j2:"
 
@@ -572,6 +598,26 @@ class Jinja2Loader(CustomLoader):
         self._template.environment.loader = self._loader
         return self._template.render(kwargs)
 
+    def get_object_to_format(self):
+        return self
+
+
+class LateYamlLoader(CustomLoader):
+    """A loader for data rendered via Jinja2, to be loaded as YAML and then deep formatted."""
+
+    def __init__(self, yaml_str, loader):
+        self._yaml_str = yaml_str
+        self._loader = loader
+
+    def get_object_to_format(self):
+        return load(self._yaml_str, search_path=self._loader._search_path)
+
+
+class Jinja2YamlLoader(Jinja2Loader):
+    def format(self, **kwargs):
+        yaml_str = super(Jinja2YamlLoader, self).format(**kwargs)
+        return LateYamlLoader(yaml_str, self)
+
 
 class CustomLoaderCollection(object):
     """Helper class to format a collection of CustomLoader objects"""

tests/yamlparser/fixtures/jinja-yaml01.xml

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <triggers class="vector">
+    <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
+      <spec/>
+      <gerritProjects>
+        <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+          <compareType>REG_EXP</compareType>
+          <pattern>a|b|c</pattern>
+          <branches>
+            <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+              <compareType>PLAIN</compareType>
+              <pattern>master</pattern>
+            </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+          </branches>
+          <disableStrictForbiddenFileVerification>false</disableStrictForbiddenFileVerification>
+        </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+        <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+          <compareType>REG_EXP</compareType>
+          <pattern>d|e|f</pattern>
+          <branches>
+            <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+              <compareType>PLAIN</compareType>
+              <pattern>stable</pattern>
+            </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+          </branches>
+          <disableStrictForbiddenFileVerification>false</disableStrictForbiddenFileVerification>
+        </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+      </gerritProjects>
+      <skipVote>
+        <onSuccessful>false</onSuccessful>
+        <onFailed>false</onFailed>
+        <onUnstable>false</onUnstable>
+        <onNotBuilt>false</onNotBuilt>
+      </skipVote>
+      <silentMode>false</silentMode>
+      <silentStartMode>false</silentStartMode>
+      <escapeQuotes>true</escapeQuotes>
+      <dependencyJobsNames/>
+      <commitMessageParameterMode>BASE64</commitMessageParameterMode>
+      <nameAndEmailParameterMode>PLAIN</nameAndEmailParameterMode>
+      <changeSubjectParameterMode>PLAIN</changeSubjectParameterMode>
+      <commentTextParameterMode>BASE64</commentTextParameterMode>
+      <notificationLevel/>
+      <dynamicTriggerConfiguration>false</dynamicTriggerConfiguration>
+      <triggerConfigURL/>
+      <dynamicGerritProjects class="empty-list"/>
+      <triggerInformationAction/>
+      <triggerOnEvents/>
+      <buildStartMessage/>
+      <buildFailureMessage/>
+      <buildSuccessfulMessage/>
+      <buildUnstableMessage/>
+      <buildNotBuiltMessage/>
+      <buildUnsuccessfulFilepath/>
+      <customUrl/>
+      <serverName>__ANY__</serverName>
+    </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
+  </triggers>
+  <builders/>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/jinja-yaml01.yaml

@@ -0,0 +1,30 @@
+- job-template:
+    name: test-job-template
+    triggers:
+    - gerrit:
+        projects:
+            !j2-yaml: |
+                {% for item in triggers %}
+                - branches:
+                  - branch-compare-type: PLAIN
+                    branch-pattern: '{{ item.branch }}'
+                  project-compare-type: REG_EXP
+                  project-pattern: '{{ item.repositories|join("|") }}'
+                {% endfor %}
+
+- project:
+    name: test-job-project
+
+    jobs:
+    - test-job-template:
+        triggers:
+        - repositories:
+            - a
+            - b
+            - c
+          branch: master
+        - repositories:
+            - d
+            - e
+            - f
+          branch: stable

tests/yamlparser/fixtures/jinja-yaml02.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild">
+      <autoRebuild>false</autoRebuild>
+      <rebuildDisabled>false</rebuildDisabled>
+    </com.sonyericsson.rebuild.RebuildSettings>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers/>
+  <buildWrappers/>
+</project>
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild">
+      <autoRebuild>false</autoRebuild>
+      <rebuildDisabled>false</rebuildDisabled>
+    </com.sonyericsson.rebuild.RebuildSettings>
+    <jenkins.model.BuildDiscarderProperty>
+      <strategy class="hudson.tasks.LogRotator">
+        <daysToKeep>7</daysToKeep>
+        <numToKeep>-1</numToKeep>
+        <artifactDaysToKeep>-1</artifactDaysToKeep>
+        <artifactNumToKeep>-1</artifactNumToKeep>
+      </strategy>
+    </jenkins.model.BuildDiscarderProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/jinja-yaml02.yaml

@@ -0,0 +1,19 @@
+- job-template:
+    name: 'test-job-{variant}'
+    properties: !j2-yaml: |
+        - rebuild
+        {% if discard_old_builds|default %}
+        - build-discarder:
+            days-to-keep: 7
+        {% endif %}
+
+- project:
+    name: test-project
+
+    jobs:
+    - 'test-job-{variant}':
+        variant: abc
+
+    - 'test-job-{variant}':
+        variant: def
+        discard_old_builds: true

tests/yamlparser/fixtures/jinja-yaml03.groovy

@@ -0,0 +1,3 @@
+if (manager.logContains(".*no_jenkins.*")) {
+    manager.build.result = hudson.model.Result.NOT_BUILT
+}

tests/yamlparser/fixtures/jinja-yaml03.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers>
+    <org.jvnet.hudson.plugins.groovypostbuild.GroovyPostbuildRecorder>
+      <behavior>0</behavior>
+      <runForMatrixParent>false</runForMatrixParent>
+      <script>
+        <script>if (manager.logContains(&quot;.*no_jenkins.*&quot;)) {
+    manager.build.result = hudson.model.Result.NOT_BUILT
+}
+</script>
+        <sandbox>false</sandbox>
+      </script>
+    </org.jvnet.hudson.plugins.groovypostbuild.GroovyPostbuildRecorder>
+  </publishers>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/jinja-yaml03.yaml

@@ -0,0 +1,15 @@
+# the purpose of this test is to check if the piece of YAML generated by
+# !j2-yaml is deep-formatted properly; if not then double quotes introduced by
+# !include-raw-escape would be left untouched and passed down to the output XML
+# file, which would simply be wrong...
+
+- job-template:
+    name: 'test-job-template'
+    publishers: !j2-yaml: |
+        - groovy-postbuild:
+            script: !include-raw-escape: ./jinja-yaml03.groovy
+
+- project:
+    name: test-project
+    jobs:
+    - 'test-job-template'