Merge "Ensure dict orders are deterministic"

jenkins_jobs/local_yaml.py

@@ -54,16 +54,58 @@ Example:
 """
 
 import codecs
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict
 import functools
 import logging
 import re
 import os
 import yaml
+from yaml.constructor import BaseConstructor
 
 logger = logging.getLogger(__name__)
 
 
-class LocalLoader(yaml.Loader):
+class OrderedConstructor(BaseConstructor):
+    """The default constructor class for PyYAML loading uses standard python
+    dictionaries which can have randomized ordering enabled (default in
+    CPython from version 3.3). The order of the XML elements being outputted
+    is both important for tests and for ensuring predictable generation based
+    on the source. This subclass overrides this behaviour to ensure that all
+    dict's created make use of OrderedDict to have iteration of keys to always
+    follow the order in which the keys were inserted/created.
+    """
+
+    def construct_yaml_map(self, node):
+        data = OrderedDict()
+        yield data
+        value = self.construct_mapping(node)
+
+        if isinstance(node, yaml.MappingNode):
+            self.flatten_mapping(node)
+        else:
+            raise yaml.constructor.ConstructorError(
+                None, None,
+                'expected a mapping node, but found %s' % node.id,
+                node.start_mark)
+
+        mapping = OrderedDict()
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=False)
+            try:
+                hash(key)
+            except TypeError as exc:
+                raise yaml.constructor.ConstructorError(
+                    'while constructing a mapping', node.start_mark,
+                    'found unacceptable key (%s)' % exc, key_node.start_mark)
+            value = self.construct_object(value_node, deep=False)
+            mapping[key] = value
+        data.update(mapping)
+
+
+class LocalLoader(OrderedConstructor, yaml.Loader):
     """Subclass for yaml.Loader which handles the local tags 'include',
     'include-raw' and 'include-raw-escaped' to specify a file to include data
     from and whether to parse it as additional yaml, treat it as a data blob
@@ -116,6 +158,11 @@ class LocalLoader(yaml.Loader):
         self.add_constructor('!include-raw-escape',
                              self._include_raw_escape_tag)
 
+        # constructor to preserve order of maps and ensure that the order of
+        # keys returned is consistent across multiple python versions
+        self.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+                             type(self).construct_yaml_map)
+
         if isinstance(self.stream, file):
             self.search_path.add(os.path.normpath(
                 os.path.dirname(self.stream.name)))

jenkins_jobs/modules/publishers.py

@@ -2399,7 +2399,7 @@ postbuildscript003.yaml
         'generic': 'GenericScript',
         'groovy': 'GroovyScriptFile',
     }
-    for script_type in script_types.keys():
+    for script_type in sorted(script_types.keys()):
         if script_type + '-script' not in data:
             continue
 

jenkins_jobs/modules/triggers.py

@@ -107,14 +107,14 @@ def build_gerrit_triggers(xml_parent, data):
 
 
 def build_gerrit_skip_votes(xml_parent, data):
-    outcomes = {'successful': 'onSuccessful',
+    outcomes = [('successful', 'onSuccessful'),
-                'failed': 'onFailed',
+                ('failed', 'onFailed'),
-                'unstable': 'onUnstable',
+                ('unstable', 'onUnstable'),
-                'notbuilt': 'onNotBuilt'}
+                ('notbuilt', 'onNotBuilt')]
 
     skip_vote_node = XML.SubElement(xml_parent, 'skipVote')
     skip_vote = data.get('skip-vote', {})
-    for result_kind, tag_name in outcomes.iteritems():
+    for result_kind, tag_name in outcomes:
         if skip_vote.get(result_kind, False):
             XML.SubElement(skip_vote_node, tag_name).text = 'true'
         else:

test-requirements.txt

@@ -2,6 +2,7 @@ hacking>=0.5.6,<0.8
 coverage>=3.6
 discover
 fixtures
+ordereddict
 python-subunit
 sphinx>=1.1.2,<1.2
 setuptools_git>=0.4

tests/builders/fixtures/ant002.xml

@@ -4,8 +4,8 @@
     <hudson.tasks.Ant>
       <buildFile>build.xml</buildFile>
       <antOpts>-ea -Xmx512m</antOpts>
-      <properties>failonerror=True
+      <properties>builddir=/tmp/
-builddir=/tmp/
+failonerror=True
 </properties>
       <targets>debug test install</targets>
       <antName>Standard Ant</antName>

tests/localyaml/fixtures/include-raw-escaped001.json

@@ -1,6 +1,7 @@
 [
     {
         "template-job": {
+            "name": "test-job-include-raw-{num}",
             "builders": [
                 {
                     "shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n"
@@ -8,17 +9,16 @@
                 {
                    "shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${{VAR1}} ${{VAR2}}\"\n\n[[ -n \"${{VAR3}}\" ]] && {{\n    # this next section is executed as one\n    echo \"${{VAR3}}\"\n    exit 0\n}}\n\n"
                 }
-            ],
+            ]
-            "name": "test-job-include-raw-{num}"
         }
     },
     {
         "project": {
+            "name": "test-job-template-1",
             "num": 1,
             "jobs": [
                 "test-job-include-raw-{num}"
-            ],
+            ]
-            "name": "test-job-template-1"
         }
     }
 ]

tests/localyaml/fixtures/include-raw001.json

@@ -1,6 +1,7 @@
 [
     {
         "job": {
+            "name": "test-job-include-raw-1",
             "builders": [
                 {
                     "shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n"
@@ -8,8 +9,7 @@
                 {
                     "shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${VAR1} ${VAR2}\"\n\n[[ -n \"${VAR3}\" ]] && {\n    # this next section is executed as one\n    echo \"${VAR3}\"\n    exit 0\n}\n\n"
                 }
-            ],
+            ]
-            "name": "test-job-include-raw-1"
         }
     }
 ]

tests/localyaml/fixtures/include001.json

@@ -1,43 +1,43 @@
 [
     {
         "job": {
+            "name": "test-job-1",
             "builders": [
                 {
                     "copyartifact": {
-                        "filter": "*.tar.gz",
                         "project": "foo",
-                        "parameter-filters": "PUBLISH=true",
+                        "filter": "*.tar.gz",
                         "target": "/home/foo",
-                        "flatten": true,
+                        "which-build": "last-successful",
                         "optional": true,
-                        "which-build": "last-successful"
+                        "flatten": true,
+                        "parameter-filters": "PUBLISH=true"
                     }
                 },
                 {
                     "copyartifact": {
                         "project": "bar",
-                        "parameter-filters": "PUBLISH=true",
-                        "target": "/home/foo",
-                        "build-number": 123,
-                        "which-build": "specific-build",
                         "filter": "*.tar.gz",
+                        "target": "/home/foo",
+                        "which-build": "specific-build",
+                        "optional": true,
                         "flatten": true,
-                        "optional": true
+                        "parameter-filters": "PUBLISH=true",
+                        "build-number": 123
                     }
                 },
                 {
                     "copyartifact": {
-                        "filter": "*.tar.gz",
                         "project": "baz",
-                        "parameter-filters": "PUBLISH=true",
+                        "filter": "*.tar.gz",
                         "target": "/home/foo",
-                        "flatten": true,
+                        "which-build": "upstream-build",
                         "optional": true,
-                        "which-build": "upstream-build"
+                        "flatten": true,
-                    }
+                        "parameter-filters": "PUBLISH=true"
-                }
+                    }
-            ],
+                }
-            "name": "test-job-1"
+            ]
         }
     }
 ]

tests/publishers/fixtures/ruby-metrics.xml

@@ -7,14 +7,14 @@
         <hudson.plugins.rubyMetrics.rcov.model.MetricTarget>
           <metric>TOTAL_COVERAGE</metric>
           <healthy>80</healthy>
-          <unstable>0</unstable>
           <unhealthy>0</unhealthy>
+          <unstable>0</unstable>
         </hudson.plugins.rubyMetrics.rcov.model.MetricTarget>
         <hudson.plugins.rubyMetrics.rcov.model.MetricTarget>
           <metric>CODE_COVERAGE</metric>
           <healthy>80</healthy>
-          <unstable>0</unstable>
           <unhealthy>0</unhealthy>
+          <unstable>0</unstable>
         </hudson.plugins.rubyMetrics.rcov.model.MetricTarget>
       </targets>
     </hudson.plugins.rubyMetrics.rcov.RcovPublisher>

tests/publishers/fixtures/xunit001.xml

@@ -18,15 +18,15 @@
       </types>
       <thresholds>
         <org.jenkinsci.plugins.xunit.threshold.FailedThreshold>
-          <failureThreshold/>
           <unstableThreshold/>
           <unstableNewThreshold/>
+          <failureThreshold/>
           <failureNewThreshold/>
         </org.jenkinsci.plugins.xunit.threshold.FailedThreshold>
         <org.jenkinsci.plugins.xunit.threshold.SkippedThreshold>
-          <failureThreshold/>
           <unstableThreshold/>
           <unstableNewThreshold/>
+          <failureThreshold/>
           <failureNewThreshold/>
         </org.jenkinsci.plugins.xunit.threshold.SkippedThreshold>
       </thresholds>