Fix default encoding issue with python 2.6

This change addresses issue #38: "fix unicode handling issues".

The issue was originally reported against neutron client
(https://bugs.launchpad.net/python-neutronclient/+bug/1189112) but
was tracked down to the fact that python 2.6 does not set the default
encoding for sys.stdout properly. A change to python 2.7 fixes the
problem there and later (http://hg.python.org/cpython/rev/e60ef17561dc/),
but since cliff supports python 2.6 it needs to handle the case
explicitly.

Change-Id: Id06507d78c7c82b25f39366ea4a6dfa4ef3a3a97

cliff/app.py

@@ -2,6 +2,8 @@
 """
 
 import argparse
+import codecs
+import locale
 import logging
 import logging.handlers
 import os
@@ -67,14 +69,31 @@ class App(object):
         """
         self.command_manager = command_manager
         self.command_manager.add_command('help', HelpCommand)
-        self.stdin = stdin or sys.stdin
+        self._set_streams(stdin, stdout, stderr)
-        self.stdout = stdout or sys.stdout
-        self.stderr = stderr or sys.stderr
         self.interactive_app_factory = interactive_app_factory
         self.parser = self.build_option_parser(description, version)
         self.interactive_mode = False
         self.interpreter = None
 
+    def _set_streams(self, stdin, stdout, stderr):
+        locale.setlocale(locale.LC_ALL, '')
+        if sys.version_info[:2] == (2, 6):
+            # Configure the input and output streams. If a stream is
+            # provided, it must be configured correctly by the
+            # caller. If not, make sure the versions of the standard
+            # streams used by default are wrapped with encodings. This
+            # works around a problem with Python 2.6 fixed in 2.7 and
+            # later (http://hg.python.org/cpython/rev/e60ef17561dc/).
+            lang, encoding = locale.getdefaultlocale()
+            encoding = getattr(sys.stdout, 'encoding', None) or encoding
+            self.stdin = stdin or codecs.getreader(encoding)(sys.stdin)
+            self.stdout = stdout or codecs.getwriter(encoding)(sys.stdout)
+            self.stderr = stderr or codecs.getwriter(encoding)(sys.stderr)
+        else:
+            self.stdin = stdin or sys.stdin
+            self.stdout = stdout or sys.stdout
+            self.stderr = stderr or sys.stderr
+
     def build_option_parser(self, description, version,
                             argparse_kwargs=None):
         """Return an argparse option parser for this application.

cliff/formatters/table.py

@@ -22,7 +22,10 @@ class TableFormatter(ListFormatter, SingleFormatter):
         pass
 
     def emit_list(self, column_names, data, stdout, parsed_args):
-        x = prettytable.PrettyTable(column_names, print_empty=False)
+        x = prettytable.PrettyTable(
+            column_names,
+            print_empty=False,
+        )
         x.padding_width = 1
         # Figure out the types of the columns in the
         # first row and set the alignment of the

cliff/tests/test_app.py

@@ -1,11 +1,19 @@
+# -*- encoding: utf-8 -*-
 from argparse import ArgumentError
+try:
+    from StringIO import StringIO
+except ImportError:
+    # Probably python 3, that test won't be run so ignore the error
+    pass
+import sys
+
+import nose
+import mock
 
 from cliff.app import App
 from cliff.command import Command
 from cliff.commandmanager import CommandManager
 
-import mock
-
 
 def make_app():
     cmd_mgr = CommandManager('cliff.tests')
@@ -227,3 +235,115 @@ def test_option_parser_conflicting_option_custom_arguments_should_not_throw():
             )
 
     MyApp()
+
+
+def test_output_encoding_default():
+    # The encoding should come from getdefaultlocale() because
+    # stdout has no encoding set.
+    if sys.version_info[:2] != (2, 6):
+        raise nose.SkipTest('only needed for python 2.6')
+    data = '\xc3\xa9'
+    u_data = data.decode('utf-8')
+
+    class MyApp(App):
+        def __init__(self):
+            super(MyApp, self).__init__(
+                description='testing',
+                version='0.1',
+                command_manager=CommandManager('tests'),
+            )
+
+    stdout = StringIO()
+    getdefaultlocale = lambda: ('ignored', 'utf-8')
+
+    with mock.patch('sys.stdout', stdout):
+        with mock.patch('locale.getdefaultlocale', getdefaultlocale):
+            app = MyApp()
+            app.stdout.write(u_data)
+            actual = stdout.getvalue()
+            assert data == actual
+
+
+def test_output_encoding_sys():
+    # The encoding should come from sys.stdout because it is set
+    # there.
+    if sys.version_info[:2] != (2, 6):
+        raise nose.SkipTest('only needed for python 2.6')
+    data = '\xc3\xa9'
+    u_data = data.decode('utf-8')
+
+    class MyApp(App):
+        def __init__(self):
+            super(MyApp, self).__init__(
+                description='testing',
+                version='0.1',
+                command_manager=CommandManager('tests'),
+            )
+
+    stdout = StringIO()
+    stdout.encoding = 'utf-8'
+    getdefaultlocale = lambda: ('ignored', 'utf-16')
+
+    with mock.patch('sys.stdout', stdout):
+        with mock.patch('locale.getdefaultlocale', getdefaultlocale):
+            app = MyApp()
+            app.stdout.write(u_data)
+            actual = stdout.getvalue()
+            assert data == actual
+
+
+def test_error_encoding_default():
+    # The encoding should come from getdefaultlocale() because
+    # stdout has no encoding set.
+    if sys.version_info[:2] != (2, 6):
+        raise nose.SkipTest('only needed for python 2.6')
+    data = '\xc3\xa9'
+    u_data = data.decode('utf-8')
+
+    class MyApp(App):
+        def __init__(self):
+            super(MyApp, self).__init__(
+                description='testing',
+                version='0.1',
+                command_manager=CommandManager('tests'),
+            )
+
+    stderr = StringIO()
+    getdefaultlocale = lambda: ('ignored', 'utf-8')
+
+    with mock.patch('sys.stderr', stderr):
+        with mock.patch('locale.getdefaultlocale', getdefaultlocale):
+            app = MyApp()
+            app.stderr.write(u_data)
+            actual = stderr.getvalue()
+            assert data == actual
+
+
+def test_error_encoding_sys():
+    # The encoding should come from sys.stdout (not sys.stderr)
+    # because it is set there.
+    if sys.version_info[:2] != (2, 6):
+        raise nose.SkipTest('only needed for python 2.6')
+    data = '\xc3\xa9'
+    u_data = data.decode('utf-8')
+
+    class MyApp(App):
+        def __init__(self):
+            super(MyApp, self).__init__(
+                description='testing',
+                version='0.1',
+                command_manager=CommandManager('tests'),
+            )
+
+    stdout = StringIO()
+    stdout.encoding = 'utf-8'
+    stderr = StringIO()
+    getdefaultlocale = lambda: ('ignored', 'utf-16')
+
+    with mock.patch('sys.stdout', stdout):
+        with mock.patch('sys.stderr', stderr):
+            with mock.patch('locale.getdefaultlocale', getdefaultlocale):
+                app = MyApp()
+                app.stderr.write(u_data)
+                actual = stderr.getvalue()
+                assert data == actual

demoapp/cliffdemo/encoding.py

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+
+import logging
+
+from cliff.lister import Lister
+
+
+class Encoding(Lister):
+    """Show some unicode text
+    """
+
+    log = logging.getLogger(__name__)
+
+    def take_action(self, parsed_args):
+        messages = [
+            u'pi: π',
+            u'GB18030:鼀丅㐀ٸཌྷᠧꌢ€',
+        ]
+        return (
+            ('UTF-8', 'Unicode'),
+            [(repr(t.encode('utf-8')), t)
+             for t in messages],
+        )

demoapp/setup.py

@@ -68,6 +68,7 @@ setup(
             'files = cliffdemo.list:Files',
             'file = cliffdemo.show:File',
             'show file = cliffdemo.show:File',
+            'unicode = cliffdemo.encoding:Encoding',
             ],
         },