Decodes input and encodes output

Currently cinderclient doesn't handle properly incoming and outgoing
encode / decode process. As a solution for this, this patch implements a
decoding process for all data incoming from the user side and encodes
everything going out of the client, i.e: http requests, prints, etc.

This patch introduces a new module (strutils.py) taken from
oslo-incubator in order to use 2 of the functions present in it:

About safe_(decode|encode):

    Both functions try to encode / decode the incoming text using the
    stdin encoding, fallback to python's default encoding if that
    returns None or to UTF-8 as the last option.

    In both functions only basestring objects are accepted and they both
    raise TypeError if an object of another type is passed.

About the general cinderclient changes:

    In order to better support non-ASCII characters, it is a good
    practice to use unicode interanlly and encode everything that has to
    go out. This patch aims to do that and introduces this behaviour in
    the client.

Testing:

    A good test (besides using tox) is to use cinder client with and
    without setting any locale (export LANG=).

Fixes bug: 1130572

Change-Id: Idb7d06954c29e003f68a0c4aa0b80ecc7017cbc9

cinderclient/openstack/common/strutils.py

@@ -0,0 +1,133 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+System-level utilities and helper functions.
+"""
+
+import logging
+import sys
+
+LOG = logging.getLogger(__name__)
+
+
+def int_from_bool_as_string(subject):
+    """
+    Interpret a string as a boolean and return either 1 or 0.
+
+    Any string value in:
+
+        ('True', 'true', 'On', 'on', '1')
+
+    is interpreted as a boolean True.
+
+    Useful for JSON-decoded stuff and config file parsing
+    """
+    return bool_from_string(subject) and 1 or 0
+
+
+def bool_from_string(subject):
+    """
+    Interpret a string as a boolean.
+
+    Any string value in:
+
+        ('True', 'true', 'On', 'on', 'Yes', 'yes', '1')
+
+    is interpreted as a boolean True.
+
+    Useful for JSON-decoded stuff and config file parsing
+    """
+    if isinstance(subject, bool):
+        return subject
+    if isinstance(subject, basestring):
+        if subject.strip().lower() in ('true', 'on', 'yes', '1'):
+            return True
+    return False
+
+
+def safe_decode(text, incoming=None, errors='strict'):
+    """
+    Decodes incoming str using `incoming` if they're
+    not already unicode.
+
+    :param incoming: Text's current encoding
+    :param errors: Errors handling policy. See here for valid
+        values http://docs.python.org/2/library/codecs.html
+    :returns: text or a unicode `incoming` encoded
+                representation of it.
+    :raises TypeError: If text is not an isntance of basestring
+    """
+    if not isinstance(text, basestring):
+        raise TypeError("%s can't be decoded" % type(text))
+
+    if isinstance(text, unicode):
+        return text
+
+    if not incoming:
+        incoming = (sys.stdin.encoding or
+                    sys.getdefaultencoding())
+
+    try:
+        return text.decode(incoming, errors)
+    except UnicodeDecodeError:
+        # Note(flaper87) If we get here, it means that
+        # sys.stdin.encoding / sys.getdefaultencoding
+        # didn't return a suitable encoding to decode
+        # text. This happens mostly when global LANG
+        # var is not set correctly and there's no
+        # default encoding. In this case, most likely
+        # python will use ASCII or ANSI encoders as
+        # default encodings but they won't be capable
+        # of decoding non-ASCII characters.
+        #
+        # Also, UTF-8 is being used since it's an ASCII
+        # extension.
+        return text.decode('utf-8', errors)
+
+
+def safe_encode(text, incoming=None,
+                encoding='utf-8', errors='strict'):
+    """
+    Encodes incoming str/unicode using `encoding`. If
+    incoming is not specified, text is expected to
+    be encoded with current python's default encoding.
+    (`sys.getdefaultencoding`)
+
+    :param incoming: Text's current encoding
+    :param encoding: Expected encoding for text (Default UTF-8)
+    :param errors: Errors handling policy. See here for valid
+        values http://docs.python.org/2/library/codecs.html
+    :returns: text or a bytestring `encoding` encoded
+                representation of it.
+    :raises TypeError: If text is not an isntance of basestring
+    """
+    if not isinstance(text, basestring):
+        raise TypeError("%s can't be encoded" % type(text))
+
+    if not incoming:
+        incoming = (sys.stdin.encoding or
+                    sys.getdefaultencoding())
+
+    if isinstance(text, unicode):
+        return text.encode(encoding, errors)
+    elif text and encoding != incoming:
+        # Decode text before encoding it with `encoding`
+        text = safe_decode(text, incoming, errors)
+        return text.encode(encoding, errors)
+
+    return text

cinderclient/shell.py

@@ -30,6 +30,7 @@ import logging
 from cinderclient import client
 from cinderclient import exceptions as exc
 import cinderclient.extension
+from cinderclient.openstack.common import strutils
 from cinderclient import utils
 from cinderclient.v1 import shell as shell_v1
 from cinderclient.v2 import shell as shell_v2
@@ -486,13 +487,16 @@ class OpenStackHelpFormatter(argparse.HelpFormatter):
 
 def main():
     try:
-        OpenStackCinderShell().main(sys.argv[1:])
+        OpenStackCinderShell().main(map(strutils.safe_decode, sys.argv[1:]))
     except KeyboardInterrupt:
         print >> sys.stderr, "... terminating cinder client"
         sys.exit(130)
     except Exception, e:
         logger.debug(e, exc_info=1)
-        print >> sys.stderr, "ERROR: %s" % e.message
+        message = e.message
+        if not isinstance(message, basestring):
+            message = str(message)
+        print >> sys.stderr, "ERROR: %s" % strutils.safe_encode(message)
         sys.exit(1)
 
 

cinderclient/utils.py

@@ -1,4 +1,3 @@
-import locale
 import os
 import re
 import sys
@@ -7,6 +6,7 @@ import uuid
 import prettytable
 
 from cinderclient import exceptions
+from cinderclient.openstack.common import strutils
 
 
 def arg(*args, **kwargs):
@@ -143,14 +143,14 @@ def print_list(objs, fields, formatters={}):
                 row.append(data)
         pt.add_row(row)
 
-    print pt.get_string(sortby=fields[0])
+    print strutils.safe_encode(pt.get_string(sortby=fields[0]))
 
 
 def print_dict(d, property="Property"):
     pt = prettytable.PrettyTable([property, 'Value'], caching=False)
     pt.aligns = ['l', 'l']
     [pt.add_row(list(r)) for r in d.iteritems()]
-    print pt.get_string(sortby=property)
+    print strutils.safe_encode(pt.get_string(sortby=property))
 
 
 def find_resource(manager, name_or_id):
@@ -164,7 +164,7 @@ def find_resource(manager, name_or_id):
 
     # now try to get entity as uuid
     try:
-        uuid.UUID(str(name_or_id))
+        uuid.UUID(strutils.safe_decode(name_or_id))
         return manager.get(name_or_id)
     except (ValueError, exceptions.NotFound):
         pass
@@ -180,11 +180,7 @@ def find_resource(manager, name_or_id):
             return manager.find(name=name_or_id)
         except exceptions.NotFound:
             try:
-                # For command-line arguments that are in Unicode
+                return manager.find(display_name=name_or_id)
-                encoding = (locale.getpreferredencoding() or
-                            sys.stdin.encoding or
-                            'UTF-8')
-                return manager.find(display_name=(name_or_id.decode(encoding)))
             except (UnicodeDecodeError, exceptions.NotFound):
                 try:
                     # Volumes does not have name, but display_name

openstack-common.conf

@@ -1,7 +1,7 @@
 [DEFAULT]
 
 # The list of modules to copy from openstack-common
-modules=setup,version
+modules=setup,version,strutils
 
 # The base module to hold the copy of openstack.common
 base=cinderclient