Merge "Handle utf8 in JJB"

jenkins_jobs/builder.py

@@ -82,7 +82,7 @@ def deep_format(obj, paramdict):
     # limitations on the values in paramdict - the post-format result must
     # still be valid YAML (so substituting-in a string containing quotes, for
     # example, is problematic).
-    if isinstance(obj, str):
+    if isinstance(obj, basestring):
         try:
             result = re.match('^{obj:(?P<key>\w+)}$', obj)
             if result is not None:
@@ -464,7 +464,7 @@ class XmlJob(object):
         return hashlib.md5(self.output()).hexdigest()
 
     def output(self):
-        out = minidom.parseString(XML.tostring(self.xml))
+        out = minidom.parseString(XML.tostring(self.xml, encoding='UTF-8'))
         return out.toprettyxml(indent='  ', encoding='utf-8')
 
 

jenkins_jobs/local_yaml.py

@@ -53,6 +53,7 @@ Example:
 
 """
 
+import codecs
 import functools
 import logging
 import re
@@ -140,7 +141,7 @@ class LocalLoader(yaml.Loader):
     def _include_raw_tag(self, loader, node):
         filename = self._find_file(loader.construct_yaml_str(node))
         try:
-            with open(filename, 'r') as f:
+            with codecs.open(filename, 'r', 'utf-8') as f:
                 data = f.read()
         except:
             logger.error("Failed to include file using search path: '{0}'"

tests/base.py

@@ -78,22 +78,24 @@ class BaseTestCase(object):
 
     logging.basicConfig()
 
-    def _read_content(self):
+    def _read_utf8_content(self):
         # Read XML content, assuming it is unicode encoded
         xml_filepath = os.path.join(self.fixtures_path, self.out_filename)
         xml_content = u"%s" % codecs.open(xml_filepath, 'r', 'utf-8').read()
+        return xml_content
 
+    def _read_yaml_content(self):
         yaml_filepath = os.path.join(self.fixtures_path, self.in_filename)
         with file(yaml_filepath, 'r') as yaml_file:
             yaml_content = yaml.load(yaml_file)
-
-        return (yaml_content, xml_content)
+        return yaml_content
 
     def test_yaml_snippet(self):
         if not self.out_filename or not self.in_filename:
             return
 
-        yaml_content, expected_xml = self._read_content()
+        expected_xml = self._read_utf8_content()
+        yaml_content = self._read_yaml_content()
         project = None
         if ('project-type' in yaml_content):
             if (yaml_content['project-type'] == "maven"):
@@ -130,11 +132,7 @@ class BaseTestCase(object):
 
 class SingleJobTestCase(BaseTestCase):
     def test_yaml_snippet(self):
-        if not self.out_filename or not self.in_filename:
-            return
-
-        xml_filepath = os.path.join(self.fixtures_path, self.out_filename)
-        expected_xml = u"%s" % open(xml_filepath, 'r').read()
+        expected_xml = self._read_utf8_content()
 
         yaml_filepath = os.path.join(self.fixtures_path, self.in_filename)
 
@@ -154,7 +152,8 @@ class SingleJobTestCase(BaseTestCase):
         parser.jobs.sort(key=operator.attrgetter('name'))
 
         # Prettify generated XML
-        pretty_xml = "\n".join(job.output() for job in parser.jobs)
+        pretty_xml = unicode("\n".join(job.output() for job in parser.jobs),
+                             'utf-8')
 
         self.assertThat(
             pretty_xml,
@@ -168,10 +167,8 @@ class SingleJobTestCase(BaseTestCase):
 class JsonTestCase(BaseTestCase):
 
     def test_yaml_snippet(self):
-        if not self.out_filename or not self.in_filename:
-            return
-
-        yaml_content, expected_json = self._read_content()
+        expected_json = self._read_utf8_content()
+        yaml_content = self._read_yaml_content()
 
         pretty_json = json.dumps(yaml_content, indent=4,
                                  separators=(',', ': '))

tests/yamlparser/fixtures/include-rawunicode001-cool.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "Unicode! ☃"

tests/yamlparser/fixtures/include-rawunicode001.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers/>
+  <buildWrappers>
+    <org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+      <buildSteps>
+        <hudson.tasks.Shell>
+          <command>#!/bin/bash
+echo &quot;Unicode! ☃&quot;
+</command>
+        </hudson.tasks.Shell>
+      </buildSteps>
+    </org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+  </buildWrappers>
+</project>

tests/yamlparser/fixtures/include-rawunicode001.yaml

@@ -0,0 +1,11 @@
+- wrapper:
+    name: unicode-raw-include-wrapper
+    wrappers:
+      - pre-scm-buildstep:
+          - shell:
+              !include-raw include-rawunicode001-cool.sh
+
+- job:
+    name: test-unicode-raw-include-wrapper
+    wrappers:
+        - unicode-raw-include-wrapper

tests/yamlparser/fixtures/unicode001.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers/>
+  <buildWrappers>
+    <org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+      <buildSteps>
+        <hudson.tasks.Shell>
+          <command>#!/bin/bash
+echo &quot;Unicode! ☃&quot;
+</command>
+        </hudson.tasks.Shell>
+      </buildSteps>
+    </org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+  </buildWrappers>
+</project>

tests/yamlparser/fixtures/unicode001.yaml

@@ -0,0 +1,12 @@
+- wrapper:
+    name: unicode-wrapper
+    wrappers:
+      - pre-scm-buildstep:
+          - shell: |
+              #!/bin/bash
+              echo "Unicode! ☃"
+
+- job:
+    name: test-unicode-wrapper
+    wrappers:
+        - unicode-wrapper