Add tests for YamlParser and patch 2.6 minidom

Test full build job XML produced by the YamlParser object which exposed differences in how the XML was being written on python 2.6 versus newer versions of python. Differences were whitespace only, but made testing and validating the xml outputted across multiple versions of python difficult. Code now patches the python 2.6 writexml method on the Element class in the xml.dom.minidom module, with a version of the method taken from python 2.7.5. Removed the now obsolete regex that was reformatting the output on python 2.6. Change-Id: I26c100b6adfbcb9b197bb06cd162855adaaf24c5
2013-12-21 20:58:05 +11:00 · 2013-12-21 20:58:05 +11:00 · b6784fcf6d
parent fedd76678b
commit b6784fcf6d
5 changed files with 323 additions and 8 deletions
--- a/jenkins_jobs/builder.py
+++ b/jenkins_jobs/builder.py
@ -16,6 +16,7 @@
 # Manage jobs in Jenkins server

 import os
+import sys
 import hashlib
 import yaml
 import json
@ -34,6 +35,41 @@ logger = logging.getLogger(__name__)
 MAGIC_MANAGE_STRING = "<!-- Managed by Jenkins Job Builder -->"


+# Python 2.6's minidom toprettyxml produces broken output by adding extraneous
+# whitespace around data. This patches the broken implementation with one taken
+# from 2.7
+def writexml(self, writer, indent="", addindent="", newl=""):
+    # indent = current indentation
+    # addindent = indentation to add to higher levels
+    # newl = newline string
+    writer.write(indent + "<" + self.tagName)
+
+    attrs = self._get_attributes()
+    a_names = attrs.keys()
+    a_names.sort()
+
+    for a_name in a_names:
+        writer.write(" %s=\"" % a_name)
+        minidom._write_data(writer, attrs[a_name].value)
+        writer.write("\"")
+    if self.childNodes:
+        writer.write(">")
+        if (len(self.childNodes) == 1 and
+                self.childNodes[0].nodeType == minidom.Node.TEXT_NODE):
+            self.childNodes[0].writexml(writer, '', '', '')
+        else:
+            writer.write(newl)
+            for node in self.childNodes:
+                node.writexml(writer, indent + addindent, addindent, newl)
+            writer.write(indent)
+        writer.write("</%s>%s" % (self.tagName, newl))
+    else:
+        writer.write("/>%s" % (newl))
+
+if sys.hexversion < 0x02070000:
+    minidom.Element.writexml = writexml
+
+
 def deep_format(obj, paramdict):
    """Apply the paramdict via str.format() to all string objects found within
       the supplied obj. Lists and dicts are traversed recursively."""
@ -239,8 +275,8 @@ class YamlParser(object):

    def getXMLForJob(self, data):
        kind = data.get('project-type', 'freestyle')
-        data["description"] = data.get("description", "") + \
-            self.get_managed_string()
+        data["description"] = (data.get("description", "") +
+                               self.get_managed_string()).lstrip()
        for ep in pkg_resources.iter_entry_points(
                group='jenkins_jobs.projects', name=kind):
            Mod = ep.load()
@ -375,14 +411,9 @@ class XmlJob(object):
    def md5(self):
        return hashlib.md5(self.output()).hexdigest()

-    # Pretty printing ideas from
-    # http://stackoverflow.com/questions/749796/pretty-printing-xml-in-python
-    pretty_text_re = re.compile('>\n\s+([^<>\s].*?)\n\s+</', re.DOTALL)
-
    def output(self):
        out = minidom.parseString(XML.tostring(self.xml))
-        out = out.toprettyxml(indent='  ', encoding='utf-8')
-        return self.pretty_text_re.sub('>\g<1></', out)
+        return out.toprettyxml(indent='  ', encoding='utf-8')


 class CacheStorage(object):
--- a/tests/yamlparser/init.py
+++ b/tests/yamlparser/init.py
--- a/tests/yamlparser/fixtures/complete001.xml
+++ b/tests/yamlparser/fixtures/complete001.xml
@ -0,0 +1,137 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.plugins.git.GitSCM">
+    <configVersion>2</configVersion>
+    <userRemoteConfigs>
+      <hudson.plugins.git.UserRemoteConfig>
+        <name>origin</name>
+        <refspec>+refs/heads/*:refs/remotes/origin/*</refspec>
+        <url>ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git</url>
+      </hudson.plugins.git.UserRemoteConfig>
+    </userRemoteConfigs>
+    <branches>
+      <hudson.plugins.git.BranchSpec>
+        <name>origin/**</name>
+      </hudson.plugins.git.BranchSpec>
+    </branches>
+    <excludedUsers/>
+    <buildChooser class="com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTriggerBuildChooser"/>
+    <disableSubmodules>false</disableSubmodules>
+    <recursiveSubmodules>false</recursiveSubmodules>
+    <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+    <authorOrCommitter>false</authorOrCommitter>
+    <clean>true</clean>
+    <wipeOutWorkspace>true</wipeOutWorkspace>
+    <pruneBranches>true</pruneBranches>
+    <remotePoll>false</remotePoll>
+    <gitTool>Default</gitTool>
+    <submoduleCfg class="list"/>
+    <relativeTargetDir/>
+    <reference/>
+    <gitConfigName/>
+    <gitConfigEmail/>
+    <skipTag>false</skipTag>
+    <scmName/>
+    <useShallowClone>false</useShallowClone>
+    <browser class="hudson.plugins.git.browser.GitWeb">
+      <url>http://review.openstack.org/gitweb?p=openstack-infra/jenkins-job-builder.git</url>
+    </browser>
+  </scm>
+  <triggers class="vector">
+    <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
+      <spec/>
+      <gerritProjects>
+        <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+          <compareType>PLAIN</compareType>
+          <pattern>openstack-infra/jenkins-job-builder</pattern>
+          <branches>
+            <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+              <compareType>ANT</compareType>
+              <pattern>**</pattern>
+            </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+          </branches>
+        </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+      </gerritProjects>
+      <skipVote>
+        <onSuccessful>false</onSuccessful>
+        <onFailed>false</onFailed>
+        <onUnstable>false</onUnstable>
+        <onNotBuilt>false</onNotBuilt>
+      </skipVote>
+      <silentMode>false</silentMode>
+      <escapeQuotes>true</escapeQuotes>
+      <noNameAndEmailParameters>false</noNameAndEmailParameters>
+      <dynamicTriggerConfiguration>False</dynamicTriggerConfiguration>
+      <triggerConfigURL/>
+      <triggerOnEvents>
+        <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.events.PluginPatchsetCreatedEvent/>
+      </triggerOnEvents>
+      <gerritBuildSuccessfulVerifiedValue>1</gerritBuildSuccessfulVerifiedValue>
+      <gerritBuildFailedVerifiedValue>-1</gerritBuildFailedVerifiedValue>
+      <buildStartMessage/>
+      <buildFailureMessage>This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset.</buildFailureMessage>
+      <buildSuccessfulMessage/>
+      <buildUnstableMessage/>
+      <customUrl/>
+    </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
+  </triggers>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>#!/usr/bin/env python
+#
+print(&quot;Doing something cool with python&quot;)
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers>
+    <hudson.plugins.build__timeout.BuildTimeoutWrapper>
+      <timeoutMinutes>3</timeoutMinutes>
+      <failBuild>true</failBuild>
+      <writingDescription>false</writingDescription>
+      <timeoutPercentage>150</timeoutPercentage>
+      <timeoutMinutesElasticDefault>90</timeoutMinutesElasticDefault>
+      <timeoutType>elastic</timeoutType>
+    </hudson.plugins.build__timeout.BuildTimeoutWrapper>
+    <org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+      <buildSteps>
+        <hudson.tasks.Shell>
+          <command>#!/bin/bash
+echo &quot;Doing somethiung cool&quot;
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Shell>
+          <command>#!/bin/zsh
+echo &quot;Doing somethin cool with zsh&quot;
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Ant>
+          <targets>target1 target2</targets>
+          <antName>default</antName>
+        </hudson.tasks.Ant>
+        <EnvInjectBuilder>
+          <info>
+            <propertiesFilePath>example.prop</propertiesFilePath>
+            <propertiesContent>EXAMPLE=foo-bar</propertiesContent>
+          </info>
+        </EnvInjectBuilder>
+      </buildSteps>
+    </org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+    <com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
+      <includes>file1,file2*.txt</includes>
+      <excludes>file2bad.txt</excludes>
+      <flatten>false</flatten>
+      <includeAntExcludes>false</includeAntExcludes>
+      <relativeTo>userContent</relativeTo>
+      <hudsonHomeRelative>false</hudsonHomeRelative>
+    </com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
+  </buildWrappers>
+</project>
--- a/tests/yamlparser/fixtures/complete001.yaml
+++ b/tests/yamlparser/fixtures/complete001.yaml
@ -0,0 +1,92 @@
+- wrapper:
+    name: timeout-wrapper
+    wrappers:
+      - timeout:
+          fail: true
+          elastic-percentage: 150
+          elastic-default-timeout: 90
+          type: elastic
+
+- wrapper:
+    name: pre-scm-shell-ant
+    wrappers:
+      - pre-scm-buildstep:
+          - shell: |
+              #!/bin/bash
+              echo "Doing somethiung cool"
+          - shell: |
+              #!/bin/zsh
+              echo "Doing somethin cool with zsh"
+          - ant: "target1 target2"
+            ant-name: "Standard Ant"
+          - inject:
+               properties-file: example.prop
+               properties-content: EXAMPLE=foo-bar
+
+- wrapper:
+    name: copy-files
+    wrappers:
+      - copy-to-slave:
+          includes:
+            - file1
+            - file2*.txt
+          excludes:
+            - file2bad.txt
+
+- trigger:
+    name: gerrit-review
+    triggers:
+      - gerrit:
+         triggerOnPatchsetUploadedEvent: true
+         triggerOnChangeMergedEvent: false
+         triggerOnRefUpdatedEvent: false
+         triggerOnCommentAddedEvent: false
+         overrideVotes: true
+         gerritBuildSuccessfulVerifiedValue: 1
+         gerritBuildFailedVerifiedValue: -1
+         projects:
+           - projectCompareType: 'PLAIN'
+             projectPattern: '{project_pattern}'
+             branchCompareType: 'ANT'
+             branchPattern: '**'
+         failureMessage: 'This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset.'
+
+- scm:
+    name: gerrit-scm
+    scm:
+      - git:
+         url: ssh://jenkins@review.openstack.org:29418/{project_pattern}.git
+         branches:
+          - origin/**
+         name: origin
+         prune: true
+         clean: true
+         browser: gitweb
+         browser-url: http://review.openstack.org/gitweb?p={project_pattern}.git
+         choosing-strategy: gerrit
+
+- project:
+    name: test
+    version:
+        - 1.2
+    jobs:
+        - 'build_myproject_{version}'
+
+- job-template:
+    name: 'build_myproject_{version}'
+    scm:
+      - gerrit-scm:
+         project_pattern: openstack-infra/jenkins-job-builder
+    triggers:
+      - gerrit-review:
+         project_pattern: openstack-infra/jenkins-job-builder
+    wrappers:
+      - timeout-wrapper
+      - pre-scm-shell-ant
+      - copy-files
+    builders:
+      - shell: |
+         #!/usr/bin/env python
+         #
+         print("Doing something cool with python")
+
--- a/tests/yamlparser/test_yamlparser.py
+++ b/tests/yamlparser/test_yamlparser.py
@ -0,0 +1,55 @@
+# Joint copyright:
+#  - Copyright 2012,2013 Wikimedia Foundation
+#  - Copyright 2012,2013 Antoine "hashar" Musso
+#  - Copyright 2013 Arnaud Fabre
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+from testtools import TestCase
+from testscenarios.testcase import TestWithScenarios
+from tests.base import get_scenarios, BaseTestCase
+import doctest
+import testtools
+from jenkins_jobs.builder import YamlParser
+
+
+class TestCaseModuleYamlInclude(TestWithScenarios, TestCase, BaseTestCase):
+    fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
+    scenarios = get_scenarios(fixtures_path)
+
+    def test_yaml_snippet(self):
+        if not self.xml_filename or not self.yaml_filename:
+            return
+
+        xml_filepath = os.path.join(self.fixtures_path, self.xml_filename)
+        expected_xml = u"%s" % open(xml_filepath, 'r').read()
+
+        yaml_filepath = os.path.join(self.fixtures_path, self.yaml_filename)
+
+        parser = YamlParser()
+        parser.parse(yaml_filepath)
+
+        # Generate the XML tree
+        parser.generateXML()
+
+        # Prettify generated XML
+        pretty_xml = parser.jobs[0].output()
+
+        self.assertThat(
+            pretty_xml,
+            testtools.matchers.DocTestMatches(expected_xml,
+                                              doctest.ELLIPSIS |
+                                              doctest.NORMALIZE_WHITESPACE |
+                                              doctest.REPORT_NDIFF)
+        )