Support lazy resolving of include yaml tags

To allow filenames referenced by the application specific yaml tags to
contain template job variables, use a lazy loading object that provides
a format method that can be called by the deep_format function.

Instead of processing the file, when a KeyError occurs on attempting to
call format on the filename after the yaml tag, create a LazyLoader
instance to wrap the data and provide a format method that can be called
at a later stage.

In order to call the correct method on the original Loader class,
LazyLoader needs to be given the custom tag class, a reference to the
loader and the node object. Using the tag class it can call the
from_yaml() method with the loader and node object to return the file
contents.

Since the result from the LazyLoader instance is triggered by calling
the format method, there is no need to escape the brackets used by
pythons format method since the output will not be passed through it.

In order to ensure this behaviour, nodes passed to the method handling
the '!include-raw-escape:' tag class, which need to use the LazyLoader
approach will convert to the '!include-raw:' tag class to the
LazyLoader initialization instead.

Due to a bug in sphinx with use of 'note' admonitions and manpage
generation, need to update to a version >= 1.2.1.

Change-Id: I187eb83ba54740c2c1b627bc99c2d9769687fbc7
Story: 2000522

jenkins_jobs/formatter.py

@@ -35,15 +35,19 @@ def deep_format(obj, paramdict, allow_empty=False):
     if hasattr(obj, 'format'):
         try:
             result = re.match('^{obj:(?P<key>\w+)}$', obj)
-            if result is not None:
-                ret = paramdict[result.group("key")]
-            else:
-                ret = CustomFormatter(allow_empty).format(obj, **paramdict)
-        except KeyError as exc:
-            missing_key = exc.args[0]
-            desc = "%s parameter missing to format %s\nGiven:\n%s" % (
-                missing_key, obj, pformat(paramdict))
-            raise JenkinsJobsException(desc)
+        except TypeError:
+            ret = obj.format(**paramdict)
+        else:
+            try:
+                if result is not None:
+                    ret = paramdict[result.group("key")]
+                else:
+                    ret = CustomFormatter(allow_empty).format(obj, **paramdict)
+            except KeyError as exc:
+                missing_key = exc.args[0]
+                desc = "%s parameter missing to format %s\nGiven:\n%s" % (
+                    missing_key, obj, pformat(paramdict))
+                raise JenkinsJobsException(desc)
     elif isinstance(obj, list):
         ret = type(obj)()
         for item in obj:

jenkins_jobs/local_yaml.py

@@ -90,6 +90,34 @@ Examples:
 For all the multi file includes, the files are simply appended using a newline
 character.
 
+
+To allow for job templates to perform substitution on the path names, when a
+filename containing a python format placeholder is encountered, lazy loading
+support is enabled, where instead of returning the contents back during yaml
+parsing, it is delayed until the variable substitution is performed.
+
+Example:
+
+    .. literalinclude:: /../../tests/yamlparser/fixtures/lazy-load-jobs001.yaml
+
+    using a list of files:
+
+    .. literalinclude::
+        /../../tests/yamlparser/fixtures/lazy-load-jobs-multi001.yaml
+
+.. note::
+
+    Because lazy-loading involves performing the substitution on the file
+    name, it means that jenkins-job-builder can not call the variable
+    substitution on the contents of the file. This means that the
+    ``!include-raw:`` tag will behave as though ``!include-raw-escape:`` tag
+    was used instead whenever name substitution on the filename is to be
+    performed.
+
+    Given the behaviour described above, when substitution is to be performed
+    on any filename passed via ``!include-raw-escape:`` the tag will be
+    automatically converted to ``!include-raw:`` and no escaping will be
+    performed.
 """
 
 import functools
@@ -249,9 +277,14 @@ class YamlInclude(BaseYAMLObject):
         return filename
 
     @classmethod
-    def _open_file(cls, loader, scalar_node):
-        filename = cls._find_file(loader.construct_yaml_str(scalar_node),
-                                  loader.search_path)
+    def _open_file(cls, loader, node):
+        node_str = loader.construct_yaml_str(node)
+        try:
+            node_str.format()
+        except KeyError:
+            return cls._lazy_load(loader, cls.yaml_tag, node)
+
+        filename = cls._find_file(node_str, loader.search_path)
         try:
             with io.open(filename, 'r', encoding='utf-8') as f:
                 return f.read()
@@ -262,18 +295,32 @@ class YamlInclude(BaseYAMLObject):
 
     @classmethod
     def _from_file(cls, loader, node):
-        data = yaml.load(cls._open_file(loader, node),
+        contents = cls._open_file(loader, node)
+        if isinstance(contents, LazyLoader):
+            return contents
+
+        data = yaml.load(contents,
                          functools.partial(cls.yaml_loader,
                                            search_path=loader.search_path))
         return data
 
+    @classmethod
+    def _lazy_load(cls, loader, tag, node_str):
+        logger.info("Lazy loading of file template '{0}' enabled"
+                    .format(node_str))
+        return LazyLoader((cls, loader, node_str))
+
     @classmethod
     def from_yaml(cls, loader, node):
         if isinstance(node, yaml.ScalarNode):
             return cls._from_file(loader, node)
         elif isinstance(node, yaml.SequenceNode):
-            return u'\n'.join(cls._from_file(loader, scalar_node)
-                              for scalar_node in node.value)
+            contents = [cls._from_file(loader, scalar_node)
+                        for scalar_node in node.value]
+            if any(isinstance(s, LazyLoader) for s in contents):
+                return LazyLoaderCollection(contents)
+
+            return u'\n'.join(contents)
         else:
             raise yaml.constructor.ConstructorError(
                 None, None, "expected either a sequence or scalar node, but "
@@ -293,7 +340,15 @@ class YamlIncludeRawEscape(YamlIncludeRaw):
 
     @classmethod
     def from_yaml(cls, loader, node):
-        return loader.escape_callback(YamlIncludeRaw.from_yaml(loader, node))
+        data = YamlIncludeRaw.from_yaml(loader, node)
+        if isinstance(data, LazyLoader):
+            logger.warn("Replacing %s tag with %s since lazy loading means "
+                        "file contents will not be deep formatted for "
+                        "variable substitution.", cls.yaml_tag,
+                        YamlIncludeRaw.yaml_tag)
+            return data
+        else:
+            return loader.escape_callback(data)
 
 
 class DeprecatedTag(BaseYAMLObject):
@@ -320,6 +375,36 @@ class YamlIncludeRawEscapeDeprecated(DeprecatedTag):
     _new = YamlIncludeRawEscape
 
 
+class LazyLoaderCollection(object):
+    """Helper class to format a collection of LazyLoader objects"""
+    def __init__(self, sequence):
+        self._data = sequence
+
+    def format(self, *args, **kwargs):
+        return u'\n'.join(item.format(*args, **kwargs) for item in self._data)
+
+
+class LazyLoader(object):
+    """Helper class to provide lazy loading of files included using !include*
+    tags where the path to the given file contains unresolved placeholders.
+    """
+
+    def __init__(self, data):
+        # str subclasses can only have one argument, so assume it is a tuple
+        # being passed and unpack as needed
+        self._cls, self._loader, self._node = data
+
+    def __str__(self):
+        return "%s %s" % (self._cls.yaml_tag, self._node.value)
+
+    def __repr__(self):
+        return "%s %s" % (self._cls.yaml_tag, self._node.value)
+
+    def format(self, *args, **kwargs):
+        self._node.value = self._node.value.format(*args, **kwargs)
+        return self._cls.from_yaml(self._loader, self._node)
+
+
 def load(stream, **kwargs):
     LocalAnchorLoader.reset_anchors()
     return yaml.load(stream, functools.partial(LocalLoader, **kwargs))

tests/yamlparser/fixtures/lazy-load-jobs-copy-files.yaml.inc

@@ -0,0 +1,8 @@
+name: copy-files
+wrappers:
+    - copy-to-slave:
+        includes:
+        - file1
+        - file2*.txt
+        excludes:
+        - file2bad.txt

tests/yamlparser/fixtures/lazy-load-jobs-multi001.xml

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>#!/bin/bash
+#
+# version 1.1 of the echo vars script
+
+MSG=&quot;hello world&quot;
+VERSION=&quot;1.1&quot;
+
+[[ -n &quot;${MSG}&quot; ]] &amp;&amp; {
+    # this next section is executed as one
+    echo &quot;${MSG}&quot;
+    echo &quot;version: ${VERSION}&quot;
+    exit 0
+}
+
+#!/bin/bash
+echo &quot;Doing somethiung cool&quot;
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers>
+    <hudson.plugins.build__timeout.BuildTimeoutWrapper>
+      <timeoutMinutes>3</timeoutMinutes>
+      <failBuild>true</failBuild>
+      <writingDescription>false</writingDescription>
+      <timeoutPercentage>150</timeoutPercentage>
+      <timeoutMinutesElasticDefault>90</timeoutMinutesElasticDefault>
+      <timeoutType>elastic</timeoutType>
+    </hudson.plugins.build__timeout.BuildTimeoutWrapper>
+  </buildWrappers>
+</project>

tests/yamlparser/fixtures/lazy-load-jobs-multi001.yaml

@@ -0,0 +1,20 @@
+- wrapper:
+    !include: lazy-load-jobs-timeout.yaml.inc
+
+- project:
+    name: test
+    num: "002"
+    version:
+      - 1.1
+    jobs:
+      - 'build_myproject_{version}'
+
+- job-template:
+    name: 'build_myproject_{version}'
+    wrappers:
+      !include: lazy-load-wrappers-{version}.yaml.inc
+    builders:
+      - shell:
+          !include-raw:
+            - lazy-load-scripts/echo_vars_{version}.sh
+            - include-raw{num}-cool.sh

tests/yamlparser/fixtures/lazy-load-jobs-pre-scm-shell-ant.yaml.inc

@@ -0,0 +1,15 @@
+name: pre-scm-shell-ant
+wrappers:
+    - pre-scm-buildstep:
+        - shell: |
+            #!/bin/bash
+            echo "Doing somethiung cool"
+        - shell: |
+            #!/bin/zsh
+            echo "Doing somethin cool with zsh"
+        - ant:
+            targets: "target1 target2"
+            ant-name: "Standard Ant"
+        - inject:
+            properties-file: example.prop
+            properties-content: EXAMPLE=foo-bar

tests/yamlparser/fixtures/lazy-load-jobs-timeout.yaml.inc

@@ -0,0 +1,7 @@
+name: timeout-wrapper
+wrappers:
+    - timeout:
+        fail: true
+        elastic-percentage: 150
+        elastic-default-timeout: 90
+        type: elastic

tests/yamlparser/fixtures/lazy-load-jobs001.conf

@@ -0,0 +1,2 @@
+[job_builder]
+include_path=tests/yamlparser/fixtures/lazy-load-scripts

tests/yamlparser/fixtures/lazy-load-jobs001.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>#!/bin/bash
+#
+# version 1.1 of the echo vars script
+
+MSG=&quot;hello world&quot;
+VERSION=&quot;1.1&quot;
+
+[[ -n &quot;${MSG}&quot; ]] &amp;&amp; {
+    # this next section is executed as one
+    echo &quot;${MSG}&quot;
+    echo &quot;version: ${VERSION}&quot;
+    exit 0
+}
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers>
+    <hudson.plugins.build__timeout.BuildTimeoutWrapper>
+      <timeoutMinutes>3</timeoutMinutes>
+      <failBuild>true</failBuild>
+      <writingDescription>false</writingDescription>
+      <timeoutPercentage>150</timeoutPercentage>
+      <timeoutMinutesElasticDefault>90</timeoutMinutesElasticDefault>
+      <timeoutType>elastic</timeoutType>
+    </hudson.plugins.build__timeout.BuildTimeoutWrapper>
+  </buildWrappers>
+</project>

tests/yamlparser/fixtures/lazy-load-jobs001.yaml

@@ -0,0 +1,17 @@
+- wrapper:
+    !include: lazy-load-jobs-timeout.yaml.inc
+
+- project:
+    name: test
+    version:
+      - 1.1
+    jobs:
+      - 'build_myproject_{version}'
+
+- job-template:
+    name: 'build_myproject_{version}'
+    wrappers:
+      !include: lazy-load-wrappers-{version}.yaml.inc
+    builders:
+      - shell:
+          !include-raw: echo_vars_{version}.sh

tests/yamlparser/fixtures/lazy-load-jobs002.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>#!/bin/bash
+#
+# version 1.2 of the echo vars script
+
+MSG=&quot;hello world&quot;
+VERSION=&quot;1.2&quot;
+
+[[ -n &quot;${MSG}&quot; ]] &amp;&amp; {
+    # this next section is executed as one
+    echo &quot;${MSG}&quot;
+    echo &quot;version: ${VERSION}&quot;
+    exit 0
+}
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers>
+    <hudson.plugins.build__timeout.BuildTimeoutWrapper>
+      <timeoutMinutes>3</timeoutMinutes>
+      <failBuild>true</failBuild>
+      <writingDescription>false</writingDescription>
+      <timeoutPercentage>150</timeoutPercentage>
+      <timeoutMinutesElasticDefault>90</timeoutMinutesElasticDefault>
+      <timeoutType>elastic</timeoutType>
+    </hudson.plugins.build__timeout.BuildTimeoutWrapper>
+    <org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+      <buildSteps>
+        <hudson.tasks.Shell>
+          <command>#!/bin/bash
+echo &quot;Doing somethiung cool&quot;
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Shell>
+          <command>#!/bin/zsh
+echo &quot;Doing somethin cool with zsh&quot;
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Ant>
+          <targets>target1 target2</targets>
+          <antName>Standard Ant</antName>
+        </hudson.tasks.Ant>
+        <EnvInjectBuilder>
+          <info>
+            <propertiesFilePath>example.prop</propertiesFilePath>
+            <propertiesContent>EXAMPLE=foo-bar</propertiesContent>
+          </info>
+        </EnvInjectBuilder>
+      </buildSteps>
+    </org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+    <com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
+      <includes>file1,file2*.txt</includes>
+      <excludes>file2bad.txt</excludes>
+      <flatten>false</flatten>
+      <includeAntExcludes>false</includeAntExcludes>
+      <relativeTo>userContent</relativeTo>
+      <hudsonHomeRelative>false</hudsonHomeRelative>
+    </com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
+  </buildWrappers>
+</project>

tests/yamlparser/fixtures/lazy-load-jobs002.yaml

@@ -0,0 +1,23 @@
+- wrapper:
+    !include lazy-load-jobs-timeout.yaml.inc
+
+- wrapper:
+    !include lazy-load-jobs-pre-scm-shell-ant.yaml.inc
+
+- wrapper:
+    !include lazy-load-jobs-copy-files.yaml.inc
+
+- project:
+    name: test
+    version:
+      - 1.2
+    jobs:
+      - 'build_myproject_{version}'
+
+- job-template:
+    name: 'build_myproject_{version}'
+    wrappers:
+      !include lazy-load-wrappers-{version}.yaml.inc
+    builders:
+      - shell:
+          !include-raw-escape lazy-load-scripts/echo_vars_{version}.sh

tests/yamlparser/fixtures/lazy-load-scripts/echo_vars_1.1.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# version 1.1 of the echo vars script
+
+MSG="hello world"
+VERSION="1.1"
+
+[[ -n "${MSG}" ]] && {
+    # this next section is executed as one
+    echo "${MSG}"
+    echo "version: ${VERSION}"
+    exit 0
+}

tests/yamlparser/fixtures/lazy-load-scripts/echo_vars_1.2.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# version 1.2 of the echo vars script
+
+MSG="hello world"
+VERSION="1.2"
+
+[[ -n "${MSG}" ]] && {
+    # this next section is executed as one
+    echo "${MSG}"
+    echo "version: ${VERSION}"
+    exit 0
+}

tests/yamlparser/fixtures/lazy-load-wrappers-1.1.yaml.inc

@@ -0,0 +1 @@
+- timeout-wrapper

tests/yamlparser/fixtures/lazy-load-wrappers-1.2.yaml.inc

@@ -0,0 +1,3 @@
+- timeout-wrapper
+- pre-scm-shell-ant
+- copy-files