Add helper tag to join arbitrary lists into strings

Adds a yaml application tag '!join:' to support defining data as a list
while having the resulting contents automatically joined together with
the specified delimiter and passed as a string into the resulting object
returned by yaml for JJB to process.

This allows users to store long lists of data that is combined together
for use with Jenkins plugins that expect a delimited string of arguments
instead of needing each module in JJB to provide individual support.

Change-Id: I745181ade3926d5c29708963189ae499a0378ece

jenkins_jobs/local_yaml.py

@@ -18,6 +18,52 @@
 """Custom application specific yamls tags are supported to provide
 enhancements when reading yaml configuration.
 
+Action Tags
+^^^^^^^^^^^
+
+These allow manipulation of data being stored in one layout in the source
+yaml for convenience and/or clarity, to another format to be processed by
+the targeted module instead of requiring all modules in JJB being capable
+of supporting multiple input formats.
+
+The tag ``!join:`` will treat the first element of the following list as
+the delimiter to use, when joining the remaining elements into a string
+and returning a single string to be consumed by the specified module option.
+
+This allows users to maintain elements of data in a list structure for ease
+of review/maintenance, and have the yaml parser convert it to a string for
+consumption as any argument for modules. The main expected use case is to
+allow for generic plugin data such as shell properties to be populated from
+a list construct which the yaml parser converts to a single string, instead
+of trying to support this within the module code which would require a
+templating engine similar to Jinja.
+
+Generic Example:
+
+    .. literalinclude:: /../../tests/localyaml/fixtures/joinlists.yaml
+
+
+Environment Inject:
+
+    .. literalinclude:: /../../tests/yamlparser/fixtures/string_join.yaml
+
+
+While this mechanism can also be used items where delimiters are supported by
+the module, that should be considered a bug that the existing code doesn't
+handle being provided a list and delimiter to perform the correct conversion
+for you. Should you discover a module that takes arguments with delimiters and
+the existing JJB codebase does not handle accepting lists, then this can be
+used as a temporary solution in place of using very long strings:
+
+Extended Params Example:
+
+    .. literalinclude::
+        /../../tests/parameters/fixtures/extended-choice-param-full.yaml
+
+
+Inclusion Tags
+^^^^^^^^^^^^^^
+
 These allow inclusion of arbitrary files as a method of having blocks of data
 managed separately to the yaml job configurations. A specific usage of this is
 inlining scripts contained in separate files, although such tags may also be
@@ -327,6 +373,25 @@ class J2String(BaseYAMLObject):
         return Jinja2Loader(node.value, loader.search_path)
 
 
+class YamlListJoin(BaseYAMLObject):
+    yaml_tag = u'!join:'
+
+    @classmethod
+    def from_yaml(cls, loader, node):
+        if isinstance(node, yaml.SequenceNode):
+            delimiter = node.value[0].value
+            if not isinstance(node.value[1], yaml.SequenceNode):
+                raise yaml.constructor.ConstructorError(
+                    None, None, "expected sequence node for join data, but "
+                                "found %s" % node.value[1].id, node.start_mark)
+
+            return delimiter.join((v.value for v in node.value[1].value))
+        else:
+            raise yaml.constructor.ConstructorError(
+                None, None, "expected sequence node, but found %s" % node.id,
+                node.start_mark)
+
+
 class YamlInclude(BaseYAMLObject):
     yaml_tag = u'!include:'
 

tests/localyaml/fixtures/joinlists.json

@@ -0,0 +1,8 @@
+[
+    {
+        "string-with-comma": "item1,item2,item3"
+    },
+    {
+        "string-with-space": "item1 item2 item3"
+    }
+]

tests/localyaml/fixtures/joinlists.yaml

@@ -0,0 +1,13 @@
+- string-with-comma: !join:
+    - ','
+    -
+        - item1
+        - item2
+        - item3
+
+- string-with-space: !join:
+    - ' '
+    -
+        - item1
+        - item2
+        - item3

tests/parameters/fixtures/extended-choice-param-full.xml

@@ -22,6 +22,44 @@
           <groovyScript/>
           <groovyClasspath/>
         </com.cwctravel.hudson.plugins.extended__choice__parameter.ExtendedChoiceParameterDefinition>
+        <com.cwctravel.hudson.plugins.extended__choice__parameter.ExtendedChoiceParameterDefinition>
+          <name>OPTIONS_CHECKBOX</name>
+          <description/>
+          <value>OptionA,OptionB,OptionC</value>
+          <visibleItemCount>2</visibleItemCount>
+          <multiSelectDelimiter>,</multiSelectDelimiter>
+          <quoteValue>false</quoteValue>
+          <defaultValue/>
+          <descriptionPropertyValue/>
+          <type>PT_CHECKBOX</type>
+          <propertyFile/>
+          <propertyKey/>
+          <defaultPropertyFile/>
+          <defaultPropertyKey/>
+          <descriptionPropertyFile/>
+          <descriptionPropertyKey/>
+          <groovyScript/>
+          <groovyClasspath/>
+        </com.cwctravel.hudson.plugins.extended__choice__parameter.ExtendedChoiceParameterDefinition>
+        <com.cwctravel.hudson.plugins.extended__choice__parameter.ExtendedChoiceParameterDefinition>
+          <name>MULTISELECTOPTIONS</name>
+          <description>Available options</description>
+          <value>foo|bar|select</value>
+          <visibleItemCount>2</visibleItemCount>
+          <multiSelectDelimiter>|</multiSelectDelimiter>
+          <quoteValue>true</quoteValue>
+          <defaultValue>foo</defaultValue>
+          <descriptionPropertyValue/>
+          <type>PT_MULTI_SELECT</type>
+          <propertyFile/>
+          <propertyKey>key</propertyKey>
+          <defaultPropertyFile/>
+          <defaultPropertyKey/>
+          <descriptionPropertyFile/>
+          <descriptionPropertyKey/>
+          <groovyScript/>
+          <groovyClasspath/>
+        </com.cwctravel.hudson.plugins.extended__choice__parameter.ExtendedChoiceParameterDefinition>
       </parameterDefinitions>
     </hudson.model.ParametersDefinitionProperty>
   </properties>

tests/parameters/fixtures/extended-choice-param-full.yaml

@@ -12,3 +12,28 @@ parameters:
       default-value: foo
       default-property-file: /home/property.prop
       default-property-key: fookey
+  - extended-choice:
+      name: OPTIONS_CHECKBOX
+      type: checkbox
+      value: !join:
+        - ','
+        -
+            - OptionA
+            - OptionB
+            - OptionC
+      visible-items: 2
+  - extended-choice:
+      name: MULTISELECTOPTIONS
+      description: "Available options"
+      property-key: key
+      quote-value: true
+      type: multi-select
+      value: !join:
+        - '|'
+        -
+            - foo
+            - bar
+            - select
+      visible-items: 2
+      multi-select-delimiter: '|'
+      default-value: foo

tests/yamlparser/fixtures/string_join.xml

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <EnvInjectJobProperty>
+      <info>
+        <propertiesContent>FILE_LIST=/path/to/file1,/path/to/file2,/path/to/file3,/path/to/file4,/path/to/file5,/path/to/file6,/path/to/file7,/path/to/file8,/path/to/file9,/path/to/file10,/path/to/file11,/path/to/file12,/path/to/file13,/path/to/file14,/path/to/file15,/path/to/file16,/path/to/file17,/path/to/file18,/path/to/file19,/path/to/file20
+</propertiesContent>
+        <loadFilesFromMaster>false</loadFilesFromMaster>
+      </info>
+      <on>true</on>
+      <keepJenkinsSystemVariables>true</keepJenkinsSystemVariables>
+      <keepBuildVariables>true</keepBuildVariables>
+      <overrideBuildParameters>false</overrideBuildParameters>
+    </EnvInjectJobProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>echo &quot;Template name: string-join-data-{name}&quot;
+echo &quot;Data to be processed:&quot;
+echo &quot;${INPUT_DATA}&quot;
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>
+<BLANKLINE>
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <EnvInjectJobProperty>
+      <info>
+        <propertiesContent>FILE_LIST=/another/different/path/to/file1,/another/different/path/to/file2,/another/different/path/to/file3,/another/different/path/to/file4,/another/different/path/to/file5,/another/different/path/to/file6,/another/different/path/to/file7,/another/different/path/to/file8,/another/different/path/to/file9,/another/different/path/to/file10,/another/different/path/to/file11,/another/different/path/to/file12,/another/different/path/to/file13,/another/different/path/to/file14,/another/different/path/to/file15,/another/different/path/to/file16,/another/different/path/to/file17,/another/different/path/to/file18,/another/different/path/to/file19,/another/different/path/to/file20
+</propertiesContent>
+        <loadFilesFromMaster>false</loadFilesFromMaster>
+      </info>
+      <on>true</on>
+      <keepJenkinsSystemVariables>true</keepJenkinsSystemVariables>
+      <keepBuildVariables>true</keepBuildVariables>
+      <overrideBuildParameters>false</overrideBuildParameters>
+    </EnvInjectJobProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>echo &quot;Template name: string-join-data-{name}&quot;
+echo &quot;Data to be processed:&quot;
+echo &quot;${INPUT_DATA}&quot;
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/string_join.yaml

@@ -0,0 +1,66 @@
+- project:
+    name: string_join_example
+    jobs:
+        - 'string-join-data-{name}':
+            name: set1
+            files: !join:
+              - ','
+              -
+                - /path/to/file1
+                - /path/to/file2
+                - /path/to/file3
+                - /path/to/file4
+                - /path/to/file5
+                - /path/to/file6
+                - /path/to/file7
+                - /path/to/file8
+                - /path/to/file9
+                - /path/to/file10
+                - /path/to/file11
+                - /path/to/file12
+                - /path/to/file13
+                - /path/to/file14
+                - /path/to/file15
+                - /path/to/file16
+                - /path/to/file17
+                - /path/to/file18
+                - /path/to/file19
+                - /path/to/file20
+        - 'string-join-data-{name}':
+            name: set2
+            files: !join:
+              - ','
+              -
+                - /another/different/path/to/file1
+                - /another/different/path/to/file2
+                - /another/different/path/to/file3
+                - /another/different/path/to/file4
+                - /another/different/path/to/file5
+                - /another/different/path/to/file6
+                - /another/different/path/to/file7
+                - /another/different/path/to/file8
+                - /another/different/path/to/file9
+                - /another/different/path/to/file10
+                - /another/different/path/to/file11
+                - /another/different/path/to/file12
+                - /another/different/path/to/file13
+                - /another/different/path/to/file14
+                - /another/different/path/to/file15
+                - /another/different/path/to/file16
+                - /another/different/path/to/file17
+                - /another/different/path/to/file18
+                - /another/different/path/to/file19
+                - /another/different/path/to/file20
+
+- job-template:
+    name: 'string-join-data-{name}'
+    properties:
+      - inject:
+          keep-system-variables: true
+          properties-content: |
+            FILE_LIST={files}
+    builders:
+      - shell: |
+         echo "Template name: {template-name}"
+         echo "Data to be processed:"
+         echo "${{INPUT_DATA}}"