From b6784fcf6de57d2c77ce8c79316582845387df05 Mon Sep 17 00:00:00 2001 From: Darragh Bailey Date: Sat, 21 Dec 2013 20:58:05 +1100 Subject: [PATCH] Add tests for YamlParser and patch 2.6 minidom Test full build job XML produced by the YamlParser object which exposed differences in how the XML was being written on python 2.6 versus newer versions of python. Differences were whitespace only, but made testing and validating the xml outputted across multiple versions of python difficult. Code now patches the python 2.6 writexml method on the Element class in the xml.dom.minidom module, with a version of the method taken from python 2.7.5. Removed the now obsolete regex that was reformatting the output on python 2.6. Change-Id: I26c100b6adfbcb9b197bb06cd162855adaaf24c5 --- jenkins_jobs/builder.py | 47 +++++-- tests/yamlparser/__init__.py | 0 tests/yamlparser/fixtures/complete001.xml | 137 +++++++++++++++++++++ tests/yamlparser/fixtures/complete001.yaml | 92 ++++++++++++++ tests/yamlparser/test_yamlparser.py | 55 +++++++++ 5 files changed, 323 insertions(+), 8 deletions(-) create mode 100644 tests/yamlparser/__init__.py create mode 100644 tests/yamlparser/fixtures/complete001.xml create mode 100644 tests/yamlparser/fixtures/complete001.yaml create mode 100644 tests/yamlparser/test_yamlparser.py diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py index 2a89653f4..0c9690436 100644 --- a/jenkins_jobs/builder.py +++ b/jenkins_jobs/builder.py @@ -16,6 +16,7 @@ # Manage jobs in Jenkins server import os +import sys import hashlib import yaml import json @@ -34,6 +35,41 @@ logger = logging.getLogger(__name__) MAGIC_MANAGE_STRING = "" +# Python 2.6's minidom toprettyxml produces broken output by adding extraneous +# whitespace around data. This patches the broken implementation with one taken +# from 2.7 +def writexml(self, writer, indent="", addindent="", newl=""): + # indent = current indentation + # addindent = indentation to add to higher levels + # newl = newline string + writer.write(indent + "<" + self.tagName) + + attrs = self._get_attributes() + a_names = attrs.keys() + a_names.sort() + + for a_name in a_names: + writer.write(" %s=\"" % a_name) + minidom._write_data(writer, attrs[a_name].value) + writer.write("\"") + if self.childNodes: + writer.write(">") + if (len(self.childNodes) == 1 and + self.childNodes[0].nodeType == minidom.Node.TEXT_NODE): + self.childNodes[0].writexml(writer, '', '', '') + else: + writer.write(newl) + for node in self.childNodes: + node.writexml(writer, indent + addindent, addindent, newl) + writer.write(indent) + writer.write("%s" % (self.tagName, newl)) + else: + writer.write("/>%s" % (newl)) + +if sys.hexversion < 0x02070000: + minidom.Element.writexml = writexml + + def deep_format(obj, paramdict): """Apply the paramdict via str.format() to all string objects found within the supplied obj. Lists and dicts are traversed recursively.""" @@ -239,8 +275,8 @@ class YamlParser(object): def getXMLForJob(self, data): kind = data.get('project-type', 'freestyle') - data["description"] = data.get("description", "") + \ - self.get_managed_string() + data["description"] = (data.get("description", "") + + self.get_managed_string()).lstrip() for ep in pkg_resources.iter_entry_points( group='jenkins_jobs.projects', name=kind): Mod = ep.load() @@ -375,14 +411,9 @@ class XmlJob(object): def md5(self): return hashlib.md5(self.output()).hexdigest() - # Pretty printing ideas from - # http://stackoverflow.com/questions/749796/pretty-printing-xml-in-python - pretty_text_re = re.compile('>\n\s+([^<>\s].*?)\n\s+\g<1> + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + + + + + origin/** + + + + + false + false + false + false + true + true + true + false + Default + + + + + + false + + false + + http://review.openstack.org/gitweb?p=openstack-infra/jenkins-job-builder.git + + + + + + + + PLAIN + openstack-infra/jenkins-job-builder + + + ANT + ** + + + + + + false + false + false + false + + false + true + false + False + + + + + 1 + -1 + + This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset. + + + + + + + + #!/usr/bin/env python +# +print("Doing something cool with python") + + + + + + + 3 + true + false + 150 + 90 + elastic + + + + + #!/bin/bash +echo "Doing somethiung cool" + + + + #!/bin/zsh +echo "Doing somethin cool with zsh" + + + + target1 target2 + default + + + + example.prop + EXAMPLE=foo-bar + + + + + + file1,file2*.txt + file2bad.txt + false + false + userContent + false + + + diff --git a/tests/yamlparser/fixtures/complete001.yaml b/tests/yamlparser/fixtures/complete001.yaml new file mode 100644 index 000000000..686b495ab --- /dev/null +++ b/tests/yamlparser/fixtures/complete001.yaml @@ -0,0 +1,92 @@ +- wrapper: + name: timeout-wrapper + wrappers: + - timeout: + fail: true + elastic-percentage: 150 + elastic-default-timeout: 90 + type: elastic + +- wrapper: + name: pre-scm-shell-ant + wrappers: + - pre-scm-buildstep: + - shell: | + #!/bin/bash + echo "Doing somethiung cool" + - shell: | + #!/bin/zsh + echo "Doing somethin cool with zsh" + - ant: "target1 target2" + ant-name: "Standard Ant" + - inject: + properties-file: example.prop + properties-content: EXAMPLE=foo-bar + +- wrapper: + name: copy-files + wrappers: + - copy-to-slave: + includes: + - file1 + - file2*.txt + excludes: + - file2bad.txt + +- trigger: + name: gerrit-review + triggers: + - gerrit: + triggerOnPatchsetUploadedEvent: true + triggerOnChangeMergedEvent: false + triggerOnRefUpdatedEvent: false + triggerOnCommentAddedEvent: false + overrideVotes: true + gerritBuildSuccessfulVerifiedValue: 1 + gerritBuildFailedVerifiedValue: -1 + projects: + - projectCompareType: 'PLAIN' + projectPattern: '{project_pattern}' + branchCompareType: 'ANT' + branchPattern: '**' + failureMessage: 'This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset.' + +- scm: + name: gerrit-scm + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/{project_pattern}.git + branches: + - origin/** + name: origin + prune: true + clean: true + browser: gitweb + browser-url: http://review.openstack.org/gitweb?p={project_pattern}.git + choosing-strategy: gerrit + +- project: + name: test + version: + - 1.2 + jobs: + - 'build_myproject_{version}' + +- job-template: + name: 'build_myproject_{version}' + scm: + - gerrit-scm: + project_pattern: openstack-infra/jenkins-job-builder + triggers: + - gerrit-review: + project_pattern: openstack-infra/jenkins-job-builder + wrappers: + - timeout-wrapper + - pre-scm-shell-ant + - copy-files + builders: + - shell: | + #!/usr/bin/env python + # + print("Doing something cool with python") + diff --git a/tests/yamlparser/test_yamlparser.py b/tests/yamlparser/test_yamlparser.py new file mode 100644 index 000000000..b70b7bbfe --- /dev/null +++ b/tests/yamlparser/test_yamlparser.py @@ -0,0 +1,55 @@ +# Joint copyright: +# - Copyright 2012,2013 Wikimedia Foundation +# - Copyright 2012,2013 Antoine "hashar" Musso +# - Copyright 2013 Arnaud Fabre +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +from testtools import TestCase +from testscenarios.testcase import TestWithScenarios +from tests.base import get_scenarios, BaseTestCase +import doctest +import testtools +from jenkins_jobs.builder import YamlParser + + +class TestCaseModuleYamlInclude(TestWithScenarios, TestCase, BaseTestCase): + fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures') + scenarios = get_scenarios(fixtures_path) + + def test_yaml_snippet(self): + if not self.xml_filename or not self.yaml_filename: + return + + xml_filepath = os.path.join(self.fixtures_path, self.xml_filename) + expected_xml = u"%s" % open(xml_filepath, 'r').read() + + yaml_filepath = os.path.join(self.fixtures_path, self.yaml_filename) + + parser = YamlParser() + parser.parse(yaml_filepath) + + # Generate the XML tree + parser.generateXML() + + # Prettify generated XML + pretty_xml = parser.jobs[0].output() + + self.assertThat( + pretty_xml, + testtools.matchers.DocTestMatches(expected_xml, + doctest.ELLIPSIS | + doctest.NORMALIZE_WHITESPACE | + doctest.REPORT_NDIFF) + )