jjb/jenkins-job-builder
Code Issues Proposed changes

Ensure dict orders are deterministic

Browse Source 
Python 3 enables hash randomization by default, additionally tox 1.7
turns on the same randomization for earlier versions of python by
default. Need to ensure that order of iteration over the yaml data and
resulting XML has deterministic order for testing.

Adapts https://gist.github.com/enaeseth/844388 which ensures data read
by yaml will have its order retained in a predictable manner across
multiple python versions.

Additionally it seems more sensible to ensure that the order of
generated XML snippets corresponding to the input yaml files are
consistently in the same order as the entries in the source files.

Closes-Bug: #1333349
Change-Id: I6bf6d298a2609cc6ddbbc6b02b7f1a04413a5c89
This commit is contained in:
Darragh Bailey 2014-07-15 14:41:59 +01:00
parent fef529287e
commit c99cbccb8e
10 changed files with 84 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
jenkins_jobs/local_yaml.py
View File

@ -54,16 +54,58 @@ Example:
"""
import codecs
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict
import functools
import logging
import re
import os
import yaml
from yaml.constructor import BaseConstructor
logger = logging.getLogger(__name__)
class LocalLoader(yaml.Loader):
class OrderedConstructor(BaseConstructor):
"""The default constructor class for PyYAML loading uses standard python
dictionaries which can have randomized ordering enabled (default in
CPython from version 3.3). The order of the XML elements being outputted
is both important for tests and for ensuring predictable generation based
on the source. This subclass overrides this behaviour to ensure that all
dict's created make use of OrderedDict to have iteration of keys to always
follow the order in which the keys were inserted/created.
"""
def construct_yaml_map(self, node):
data = OrderedDict()
yield data
value = self.construct_mapping(node)
if isinstance(node, yaml.MappingNode):
self.flatten_mapping(node)
else:
raise yaml.constructor.ConstructorError(
None, None,
'expected a mapping node, but found %s' % node.id,
node.start_mark)
mapping = OrderedDict()
for key_node, value_node in node.value:
key = self.construct_object(key_node, deep=False)
try:
hash(key)
except TypeError as exc:
raise yaml.constructor.ConstructorError(
'while constructing a mapping', node.start_mark,
'found unacceptable key (%s)' % exc, key_node.start_mark)
value = self.construct_object(value_node, deep=False)
mapping[key] = value
data.update(mapping)
class LocalLoader(OrderedConstructor, yaml.Loader):
"""Subclass for yaml.Loader which handles the local tags 'include',
'include-raw' and 'include-raw-escaped' to specify a file to include data
from and whether to parse it as additional yaml, treat it as a data blob
@ -116,6 +158,11 @@ class LocalLoader(yaml.Loader):
self.add_constructor('!include-raw-escape',
self._include_raw_escape_tag)
# constructor to preserve order of maps and ensure that the order of
# keys returned is consistent across multiple python versions
self.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
type(self).construct_yaml_map)
if isinstance(self.stream, file):
self.search_path.add(os.path.normpath(
os.path.dirname(self.stream.name)))

2
jenkins_jobs/modules/publishers.py
View File

@ -2399,7 +2399,7 @@ postbuildscript003.yaml
'generic': 'GenericScript',
'groovy': 'GroovyScriptFile',
}
for script_type in script_types.keys():
for script_type in sorted(script_types.keys()):
if script_type + '-script' not in data:
continue

10
jenkins_jobs/modules/triggers.py
View File

@ -107,14 +107,14 @@ def build_gerrit_triggers(xml_parent, data):
def build_gerrit_skip_votes(xml_parent, data):
outcomes = {'successful': 'onSuccessful',
'failed': 'onFailed',
'unstable': 'onUnstable',
'notbuilt': 'onNotBuilt'}
outcomes = [('successful', 'onSuccessful'),
('failed', 'onFailed'),
('unstable', 'onUnstable'),
('notbuilt', 'onNotBuilt')]
skip_vote_node = XML.SubElement(xml_parent, 'skipVote')
skip_vote = data.get('skip-vote', {})
for result_kind, tag_name in outcomes.iteritems():
for result_kind, tag_name in outcomes:
if skip_vote.get(result_kind, False):
XML.SubElement(skip_vote_node, tag_name).text = 'true'
else:

1
test-requirements.txt
View File

@ -2,6 +2,7 @@ hacking>=0.5.6,<0.8
coverage>=3.6
discover
fixtures
ordereddict
python-subunit
sphinx>=1.1.2,<1.2
setuptools_git>=0.4

4
tests/builders/fixtures/ant002.xml
View File

@ -4,8 +4,8 @@
<hudson.tasks.Ant>
<buildFile>build.xml</buildFile>
<antOpts>-ea -Xmx512m</antOpts>
<properties>failonerror=True
builddir=/tmp/
<properties>builddir=/tmp/
failonerror=True
</properties>
<targets>debug test install</targets>
<antName>Standard Ant</antName>

8
tests/localyaml/fixtures/include-raw-escaped001.json
View File

@ -1,6 +1,7 @@
[
{
"template-job": {
"name": "test-job-include-raw-{num}",
"builders": [
{
"shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n"
@ -8,17 +9,16 @@
{
"shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${{VAR1}} ${{VAR2}}\"\n\n[[ -n \"${{VAR3}}\" ]] && {{\n # this next section is executed as one\n echo \"${{VAR3}}\"\n exit 0\n}}\n\n"
}
],
"name": "test-job-include-raw-{num}"
]
}
},
{
"project": {
"name": "test-job-template-1",
"num": 1,
"jobs": [
"test-job-include-raw-{num}"
],
"name": "test-job-template-1"
]
}
}
]

4
tests/localyaml/fixtures/include-raw001.json
View File

@ -1,6 +1,7 @@
[
{
"job": {
"name": "test-job-include-raw-1",
"builders": [
{
"shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n"
@ -8,8 +9,7 @@
{
"shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${VAR1} ${VAR2}\"\n\n[[ -n \"${VAR3}\" ]] && {\n # this next section is executed as one\n echo \"${VAR3}\"\n exit 0\n}\n\n"
}
],
"name": "test-job-include-raw-1"
]
}
}
]

30
tests/localyaml/fixtures/include001.json
View File

@ -1,43 +1,43 @@
[
{
"job": {
"name": "test-job-1",
"builders": [
{
"copyartifact": {
"filter": "*.tar.gz",
"project": "foo",
"parameter-filters": "PUBLISH=true",
"filter": "*.tar.gz",
"target": "/home/foo",
"flatten": true,
"which-build": "last-successful",
"optional": true,
"which-build": "last-successful"
"flatten": true,
"parameter-filters": "PUBLISH=true"
}
},
{
"copyartifact": {
"project": "bar",
"parameter-filters": "PUBLISH=true",
"target": "/home/foo",
"build-number": 123,
"which-build": "specific-build",
"filter": "*.tar.gz",
"target": "/home/foo",
"which-build": "specific-build",
"optional": true,
"flatten": true,
"optional": true
"parameter-filters": "PUBLISH=true",
"build-number": 123
}
},
{
"copyartifact": {
"filter": "*.tar.gz",
"project": "baz",
"parameter-filters": "PUBLISH=true",
"filter": "*.tar.gz",
"target": "/home/foo",
"flatten": true,
"which-build": "upstream-build",
"optional": true,
"which-build": "upstream-build"
"flatten": true,
"parameter-filters": "PUBLISH=true"
}
}
],
"name": "test-job-1"
]
}
}
]

4
tests/publishers/fixtures/ruby-metrics.xml
View File

@ -7,14 +7,14 @@
<hudson.plugins.rubyMetrics.rcov.model.MetricTarget>
<metric>TOTAL_COVERAGE</metric>
<healthy>80</healthy>
<unstable>0</unstable>
<unhealthy>0</unhealthy>
<unstable>0</unstable>
</hudson.plugins.rubyMetrics.rcov.model.MetricTarget>
<hudson.plugins.rubyMetrics.rcov.model.MetricTarget>
<metric>CODE_COVERAGE</metric>
<healthy>80</healthy>
<unstable>0</unstable>
<unhealthy>0</unhealthy>
<unstable>0</unstable>
</hudson.plugins.rubyMetrics.rcov.model.MetricTarget>
</targets>
</hudson.plugins.rubyMetrics.rcov.RcovPublisher>

4
tests/publishers/fixtures/xunit001.xml
View File

@ -18,15 +18,15 @@
</types>
<thresholds>
<org.jenkinsci.plugins.xunit.threshold.FailedThreshold>
<failureThreshold/>
<unstableThreshold/>
<unstableNewThreshold/>
<failureThreshold/>
<failureNewThreshold/>
</org.jenkinsci.plugins.xunit.threshold.FailedThreshold>
<org.jenkinsci.plugins.xunit.threshold.SkippedThreshold>
<failureThreshold/>
<unstableThreshold/>
<unstableNewThreshold/>
<failureThreshold/>
<failureNewThreshold/>
</org.jenkinsci.plugins.xunit.threshold.SkippedThreshold>
</thresholds>