Added possibility to use non-existent keys

Added a new configuration option under the section "job_builder" named
"allow_empty_variables" that if set to true, will replace non existing variables
in strings with the empty string instead of raising an error.

It's very useful if you  have a shell script, that has optional values, for
example:

 EXTRA_PACKAGES=({extra-packages})
 for package in "${EXTRA_PACKAGES[@]}"; do
    install "$package"
 done

Or modifying the script behavior with flags, with empty as default value:

 WITH_AUTOGEN={with-autogen}
 if [[ $WITH_AUTOGEN ]]; then
    ./autogen.sh
 fi

Then if you have two different jobs that use that script in their builder, you
just have to set the extra parameter or ignore it to change the behavior:

 - builder:
   name: mybuilder
   builders:
      - shell: !include shell-scripts/myscript.sh

 - job-template:
   name: 'mytpl-{name}'
   ...
   builders:
     - mybuilder

 - project:
   name: myproj1
   with-autogen: true
   jobs:
     - 'mytpl-{name}'

 - project:
   name: myproj2
   extra-packages: |
     extrapkg1
     extrapkg2

Change-Id: Iad9f0e522725e6fd6681cd62d3e36f69baf09585
Signed-off-by: David Caro <dcaroest@redhat.com>

doc/source/definition.rst

@@ -261,6 +261,18 @@ For example:
 .. literalinclude:: /../../tests/yamlparser/fixtures/second_order_parameter_interpolation002.yaml
 
 
+By default JJB will fail if it tries to interpolate a variable that was not
+defined, but you can change that behaviour and allow empty variables with the
+allow_empty_variables configuration option.
+
+For example, having a configuration file with tha toption enabled:
+
+.. literalinclude:: /../../tests/yamlparser/fixtures/allow_empty_variables.conf
+
+Will prevent JJb from failing if there are any non-initialized variables used
+and replace them with the empty string instead.
+
+
 Yaml Anchors & Aliases
 ^^^^^^^^^^^^^^^^^^^^^^
 

doc/source/installation.rst

@@ -88,6 +88,14 @@ job_builder section
   correct one to use. When this option is set to True, only a warning is
   emitted.
 
+**allow_empty_variables**
+  (Optional) When expanding strings, by default `jenkins-jobs` will raise an
+  exception if there's a key in the string, that has not been declared on the
+  yamls. Setting this options to True, will replace it with the empty string,
+  allowing you to use those strings without having to define all the keys it
+  might be using.
+
+
 jenkins section
 ^^^^^^^^^^^^^^^
 

jenkins_jobs/builder.py

@@ -32,6 +32,7 @@ import logging
 import copy
 import itertools
 import fnmatch
+from string import Formatter
 from jenkins_jobs.errors import JenkinsJobsException
 import jenkins_jobs.local_yaml as local_yaml
 
@@ -39,6 +40,28 @@ logger = logging.getLogger(__name__)
 MAGIC_MANAGE_STRING = "<!-- Managed by Jenkins Job Builder -->"
 
 
+class CustomFormatter(Formatter):
+    """
+    Custom formatter to allow non-existing key references when formatting a
+    string
+    """
+    def __init__(self, allow_empty=False):
+        super(CustomFormatter, self).__init__()
+        self.allow_empty = allow_empty
+
+    def get_value(self, key, args, kwargs):
+        try:
+            return Formatter.get_value(self, key, args, kwargs)
+        except KeyError:
+            if self.allow_empty:
+                logger.debug(
+                    'Found uninitialized key %s, replaced with empty string',
+                    key
+                )
+                return ''
+            raise
+
+
 # Python 2.6's minidom toprettyxml produces broken output by adding extraneous
 # whitespace around data. This patches the broken implementation with one taken
 # from Python > 2.7.3
@@ -76,7 +99,7 @@ if sys.version_info[:3] < (2, 7, 3) or xml.__name__ != 'xml':
     minidom.Element.writexml = writexml
 
 
-def deep_format(obj, paramdict):
+def deep_format(obj, paramdict, allow_empty=False):
     """Apply the paramdict via str.format() to all string objects found within
        the supplied obj. Lists and dicts are traversed recursively."""
     # YAML serialisation was originally used to achieve this, but that places
@@ -89,7 +112,7 @@ def deep_format(obj, paramdict):
             if result is not None:
                 ret = paramdict[result.group("key")]
             else:
-                ret = obj.format(**paramdict)
+                ret = CustomFormatter(allow_empty).format(obj, **paramdict)
         except KeyError as exc:
             missing_key = exc.message
             desc = "%s parameter missing to format %s\nGiven:\n%s" % (
@@ -98,13 +121,13 @@ def deep_format(obj, paramdict):
     elif isinstance(obj, list):
         ret = []
         for item in obj:
-            ret.append(deep_format(item, paramdict))
+            ret.append(deep_format(item, paramdict, allow_empty))
     elif isinstance(obj, dict):
         ret = {}
         for item in obj:
             try:
-                ret[item.format(**paramdict)] = \
-                    deep_format(obj[item], paramdict)
+                ret[CustomFormatter(allow_empty).format(item, **paramdict)] = \
+                    deep_format(obj[item], paramdict, allow_empty)
             except KeyError as exc:
                 missing_key = exc.message
                 desc = "%s parameter missing to format %s\nGiven:\n%s" % (
@@ -364,7 +387,13 @@ class YamlParser(object):
 
             params.update(expanded_values)
             params = deep_format(params, params)
-            expanded = deep_format(template, params)
+            allow_empty_variables = self.config \
+                and self.config.has_section('job_builder') \
+                and self.config.has_option(
+                    'job_builder', 'allow_empty_variables') \
+                and self.config.getboolean(
+                    'job_builder', 'allow_empty_variables')
+            expanded = deep_format(template, params, allow_empty_variables)
 
             job_name = expanded.get('name')
             if jobs_glob and not matches(job_name, jobs_glob):
@@ -526,7 +555,14 @@ class ModuleRegistry(object):
                 # Template data contains values that should be interpolated
                 # into the component definition
                 s = yaml.dump(component_data, default_flow_style=False)
-                s = s.format(**template_data)
+                allow_empty_variables = self.global_config \
+                    and self.global_config.has_section('job_builder') \
+                    and self.global_config.has_option(
+                        'job_builder', 'allow_empty_variables') \
+                    and self.global_config.getboolean(
+                        'job_builder', 'allow_empty_variables')
+                s = CustomFormatter(
+                    allow_empty_variables).format(s, **template_data)
                 component_data = yaml.load(s)
         else:
             # The component is a simple string name, eg "run-tests"

jenkins_jobs/cmd.py

@@ -36,6 +36,7 @@ ignore_cache=False
 recursive=False
 exclude=.*
 allow_duplicates=False
+allow_empty_variables=False
 
 [jenkins]
 url=http://localhost:8080/
@@ -144,6 +145,11 @@ def create_parser():
     parser.add_argument('--version', dest='version', action='version',
                         version=version(),
                         help='show version')
+    parser.add_argument(
+        '--allow-empty-variables', action='store_true',
+        dest='allow_empty_variables', default=None,
+        help='Don\'t fail if any of the variables inside any string are not '
+        'defined, replace with empty string instead')
 
     return parser
 
@@ -232,6 +238,10 @@ def execute(options, config):
         if not isinstance(plugins_info, list):
             raise JenkinsJobsException("{0} must contain a Yaml list!"
                                        .format(options.plugins_info_path))
+    if options.allow_empty_variables is not None:
+        config.set('job_builder',
+                   'allow_empty_variables',
+                   str(options.allow_empty_variables))
 
     builder = Builder(config.get('jenkins', 'url'),
                       user,

tests/yamlparser/fixtures/allow_empty_variables.conf

@@ -0,0 +1,2 @@
+[job_builder]
+allow_empty_variables = True

tests/yamlparser/fixtures/allow_empty_variables.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>echo &quot;This should be empty: &quot;
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/allow_empty_variables.yaml

@@ -0,0 +1,10 @@
+- project:
+    name: allow_empty_variables
+    jobs:
+        - 'allow_empty_variables'
+
+- job-template:
+    name: 'allow_empty_variables'
+    builders:
+      - shell: |
+         echo "This should be empty: {my_empty_var}"

tests/yamlparser/fixtures/allow_empty_variables_include.conf

@@ -0,0 +1,2 @@
+[job_builder]
+allow_empty_variables = true

tests/yamlparser/fixtures/allow_empty_variables_include.sh

@@ -0,0 +1 @@
+echo "Here ->{myvar}<- you should see nothing"

tests/yamlparser/fixtures/allow_empty_variables_include.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>echo &quot;Here -&gt;&lt;- you should see nothing&quot;</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/allow_empty_variables_include.yaml

@@ -0,0 +1,11 @@
+- project:
+    name: allow_empty_variables_include
+    jobs:
+      - 'allow_empty_variables_include'
+
+- job-template:
+    name: allow_empty_variables_include
+    builders:
+      - shell:
+          !include ./allow_empty_variables_include.sh
+