Replace utils.ensure_(str|unicode) with strutils.safe(decode|encode)

Glanceclient implemented both functions before they landed into oslo.
Since both functions are already in oslo, it is now possible to pull
them in.

There's a small difference between glance's implementation and oslo's,
that is the later does not convert non-str objects - int, bool - to str
before trying to decode / encode them. This patch takes care of that
where necessary, more precisely, while encoding headers before doing a
new request.

Fixes bug: #1172253

Change-Id: I9a0dca31140bae28d8ec6aede515c5bb852b701b

glanceclient/common/http.py

@@ -36,6 +36,7 @@ import OpenSSL
 
 from glanceclient import exc
 from glanceclient.common import utils
+from glanceclient.openstack.common import strutils
 
 try:
     from eventlet import patcher
@@ -130,7 +131,7 @@ class HTTPClient(object):
             curl.append('-d \'%s\'' % kwargs['body'])
 
         curl.append('%s%s' % (self.endpoint, url))
-        LOG.debug(utils.ensure_str(' '.join(curl)))
+        LOG.debug(strutils.safe_encode(' '.join(curl)))
 
     @staticmethod
     def log_http_response(resp, body=None):
@@ -140,7 +141,7 @@ class HTTPClient(object):
         dump.append('')
         if body:
             dump.extend([body, ''])
-        LOG.debug(utils.ensure_str('\n'.join(dump)))
+        LOG.debug(strutils.safe_encode('\n'.join(dump)))
 
     @staticmethod
     def encode_headers(headers):
@@ -154,7 +155,7 @@ class HTTPClient(object):
         :returns: Dictionary with encoded headers'
                   names and values
         """
-        to_str = utils.ensure_str
+        to_str = strutils.safe_encode
         return dict([(to_str(h), to_str(v)) for h, v in headers.iteritems()])
 
     def _http_request(self, url, method, **kwargs):
@@ -182,7 +183,7 @@ class HTTPClient(object):
             conn_url = posixpath.normpath('%s/%s' % (self.endpoint_path, url))
             # Note(flaper87): Ditto, headers / url
             # encoding to make httplib happy.
-            conn_url = utils.ensure_str(conn_url)
+            conn_url = strutils.safe_encode(conn_url)
             if kwargs['headers'].get('Transfer-Encoding') == 'chunked':
                 conn.putrequest(method, conn_url)
                 for header, value in kwargs['headers'].items():

glanceclient/common/utils.py

@@ -23,6 +23,7 @@ import prettytable
 
 from glanceclient import exc
 from glanceclient.openstack.common import importutils
+from glanceclient.openstack.common import strutils
 
 
 # Decorator for cli-args
@@ -54,14 +55,14 @@ def print_list(objs, fields, formatters={}):
                 row.append(data)
         pt.add_row(row)
 
-    print ensure_str(pt.get_string())
+    print strutils.safe_encode(pt.get_string())
 
 
 def print_dict(d):
     pt = prettytable.PrettyTable(['Property', 'Value'], caching=False)
     pt.align = 'l'
     [pt.add_row(list(r)) for r in d.iteritems()]
-    print ensure_str(pt.get_string(sortby='Property'))
+    print strutils.safe_encode(pt.get_string(sortby='Property'))
 
 
 def find_resource(manager, name_or_id):
@@ -75,7 +76,7 @@ def find_resource(manager, name_or_id):
 
     # now try to get entity as uuid
     try:
-        uuid.UUID(ensure_str(name_or_id))
+        uuid.UUID(strutils.safe_encode(name_or_id))
         return manager.get(name_or_id)
     except (ValueError, exc.NotFound):
         pass
@@ -137,7 +138,7 @@ def import_versioned_module(version, submodule=None):
 
 def exit(msg=''):
     if msg:
-        print >> sys.stderr, ensure_str(msg)
+        print >> sys.stderr, strutils.safe_encode(msg)
     sys.exit(1)
 
 
@@ -192,79 +193,6 @@ def make_size_human_readable(size):
     return '%s%s' % (stripped, suffix[index])
 
 
-def ensure_unicode(text, incoming=None, errors='strict'):
-    """
-    Decodes incoming objects using `incoming` if they're
-    not already unicode.
-
-    :param incoming: Text's current encoding
-    :param errors: Errors handling policy.
-    :returns: text or a unicode `incoming` encoded
-                representation of it.
-    """
-    if isinstance(text, unicode):
-        return text
-
-    if not incoming:
-        incoming = sys.stdin.encoding or \
-            sys.getdefaultencoding()
-
-    # Calling `str` in case text is a non str
-    # object.
-    text = str(text)
-    try:
-        return text.decode(incoming, errors)
-    except UnicodeDecodeError:
-        # Note(flaper87) If we get here, it means that
-        # sys.stdin.encoding / sys.getdefaultencoding
-        # didn't return a suitable encoding to decode
-        # text. This happens mostly when global LANG
-        # var is not set correctly and there's no
-        # default encoding. In this case, most likely
-        # python will use ASCII or ANSI encoders as
-        # default encodings but they won't be capable
-        # of decoding non-ASCII characters.
-        #
-        # Also, UTF-8 is being used since it's an ASCII
-        # extension.
-        return text.decode('utf-8', errors)
-
-
-def ensure_str(text, incoming=None,
-               encoding='utf-8', errors='strict'):
-    """
-    Encodes incoming objects using `encoding`. If
-    incoming is not specified, text is expected to
-    be encoded with current python's default encoding.
-    (`sys.getdefaultencoding`)
-
-    :param incoming: Text's current encoding
-    :param encoding: Expected encoding for text (Default UTF-8)
-    :param errors: Errors handling policy.
-    :returns: text or a bytestring `encoding` encoded
-                representation of it.
-    """
-
-    if not incoming:
-        incoming = sys.stdin.encoding or \
-            sys.getdefaultencoding()
-
-    if not isinstance(text, basestring):
-        # try to convert `text` to string
-        # This allows this method for receiving
-        # objs that can be converted to string
-        text = str(text)
-
-    if isinstance(text, unicode):
-        return text.encode(encoding, errors)
-    elif text and encoding != incoming:
-        # Decode text before encoding it with `encoding`
-        text = ensure_unicode(text, incoming, errors)
-        return text.encode(encoding, errors)
-
-    return text
-
-
 def getsockopt(self, *args, **kwargs):
     """
     A function which allows us to monkey patch eventlet's

glanceclient/openstack/common/gettextutils.py

@@ -0,0 +1,50 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+gettext for openstack-common modules.
+
+Usual usage in an openstack.common module:
+
+    from glanceclient.openstack.common.gettextutils import _
+"""
+
+import gettext
+import os
+
+_localedir = os.environ.get('glanceclient'.upper() + '_LOCALEDIR')
+_t = gettext.translation('glanceclient', localedir=_localedir, fallback=True)
+
+
+def _(msg):
+    return _t.ugettext(msg)
+
+
+def install(domain):
+    """Install a _() function using the given translation domain.
+
+    Given a translation domain, install a _() function using gettext's
+    install() function.
+
+    The main difference from gettext.install() is that we allow
+    overriding the default localedir (e.g. /usr/share/locale) using
+    a translation-domain-specific environment variable (e.g.
+    NOVA_LOCALEDIR).
+    """
+    gettext.install(domain,
+                    localedir=os.environ.get(domain.upper() + '_LOCALEDIR'),
+                    unicode=True)

glanceclient/openstack/common/strutils.py

@@ -0,0 +1,150 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+System-level utilities and helper functions.
+"""
+
+import sys
+
+from glanceclient.openstack.common.gettextutils import _
+
+
+TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
+FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
+
+
+def int_from_bool_as_string(subject):
+    """
+    Interpret a string as a boolean and return either 1 or 0.
+
+    Any string value in:
+
+        ('True', 'true', 'On', 'on', '1')
+
+    is interpreted as a boolean True.
+
+    Useful for JSON-decoded stuff and config file parsing
+    """
+    return bool_from_string(subject) and 1 or 0
+
+
+def bool_from_string(subject, strict=False):
+    """
+    Interpret a string as a boolean.
+
+    A case-insensitive match is performed such that strings matching 't',
+    'true', 'on', 'y', 'yes', or '1' are considered True and, when
+    `strict=False`, anything else is considered False.
+
+    Useful for JSON-decoded stuff and config file parsing.
+
+    If `strict=True`, unrecognized values, including None, will raise a
+    ValueError which is useful when parsing values passed in from an API call.
+    Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
+    """
+    if not isinstance(subject, basestring):
+        subject = str(subject)
+
+    lowered = subject.strip().lower()
+
+    if lowered in TRUE_STRINGS:
+        return True
+    elif lowered in FALSE_STRINGS:
+        return False
+    elif strict:
+        acceptable = ', '.join(
+            "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
+        msg = _("Unrecognized value '%(val)s', acceptable values are:"
+                " %(acceptable)s") % {'val': subject,
+                                      'acceptable': acceptable}
+        raise ValueError(msg)
+    else:
+        return False
+
+
+def safe_decode(text, incoming=None, errors='strict'):
+    """
+    Decodes incoming str using `incoming` if they're
+    not already unicode.
+
+    :param incoming: Text's current encoding
+    :param errors: Errors handling policy. See here for valid
+        values http://docs.python.org/2/library/codecs.html
+    :returns: text or a unicode `incoming` encoded
+                representation of it.
+    :raises TypeError: If text is not an isntance of basestring
+    """
+    if not isinstance(text, basestring):
+        raise TypeError("%s can't be decoded" % type(text))
+
+    if isinstance(text, unicode):
+        return text
+
+    if not incoming:
+        incoming = (sys.stdin.encoding or
+                    sys.getdefaultencoding())
+
+    try:
+        return text.decode(incoming, errors)
+    except UnicodeDecodeError:
+        # Note(flaper87) If we get here, it means that
+        # sys.stdin.encoding / sys.getdefaultencoding
+        # didn't return a suitable encoding to decode
+        # text. This happens mostly when global LANG
+        # var is not set correctly and there's no
+        # default encoding. In this case, most likely
+        # python will use ASCII or ANSI encoders as
+        # default encodings but they won't be capable
+        # of decoding non-ASCII characters.
+        #
+        # Also, UTF-8 is being used since it's an ASCII
+        # extension.
+        return text.decode('utf-8', errors)
+
+
+def safe_encode(text, incoming=None,
+                encoding='utf-8', errors='strict'):
+    """
+    Encodes incoming str/unicode using `encoding`. If
+    incoming is not specified, text is expected to
+    be encoded with current python's default encoding.
+    (`sys.getdefaultencoding`)
+
+    :param incoming: Text's current encoding
+    :param encoding: Expected encoding for text (Default UTF-8)
+    :param errors: Errors handling policy. See here for valid
+        values http://docs.python.org/2/library/codecs.html
+    :returns: text or a bytestring `encoding` encoded
+                representation of it.
+    :raises TypeError: If text is not an isntance of basestring
+    """
+    if not isinstance(text, basestring):
+        raise TypeError("%s can't be encoded" % type(text))
+
+    if not incoming:
+        incoming = (sys.stdin.encoding or
+                    sys.getdefaultencoding())
+
+    if isinstance(text, unicode):
+        return text.encode(encoding, errors)
+    elif text and encoding != incoming:
+        # Decode text before encoding it with `encoding`
+        text = safe_decode(text, incoming, errors)
+        return text.encode(encoding, errors)
+
+    return text

glanceclient/shell.py

@@ -27,6 +27,7 @@ from keystoneclient.v2_0 import client as ksclient
 import glanceclient
 from glanceclient import exc
 from glanceclient.common import utils
+from glanceclient.openstack.common import strutils
 
 
 class OpenStackImagesShell(object):
@@ -466,7 +467,7 @@ class HelpFormatter(argparse.HelpFormatter):
 
 def main():
     try:
-        OpenStackImagesShell().main(map(utils.ensure_unicode, sys.argv[1:]))
+        OpenStackImagesShell().main(map(strutils.safe_decode, sys.argv[1:]))
     except KeyboardInterrupt:
         print >> sys.stderr, '... terminating glance client'
         sys.exit(1)

glanceclient/v1/images.py

@@ -21,6 +21,7 @@ import urllib
 
 from glanceclient.common import base
 from glanceclient.common import utils
+from glanceclient.openstack.common import strutils
 
 UPDATE_PARAMS = ('name', 'disk_format', 'container_format', 'min_disk',
                  'min_ram', 'owner', 'size', 'is_public', 'protected',
@@ -58,14 +59,14 @@ class ImageManager(base.Manager):
 
     def _image_meta_from_headers(self, headers):
         meta = {'properties': {}}
-        ensure_unicode = utils.ensure_unicode
+        safe_decode = strutils.safe_decode
         for key, value in headers.iteritems():
-            value = ensure_unicode(value, incoming='utf-8')
+            value = safe_decode(value, incoming='utf-8')
             if key.startswith('x-image-meta-property-'):
-                _key = ensure_unicode(key[22:], incoming='utf-8')
+                _key = safe_decode(key[22:], incoming='utf-8')
                 meta['properties'][_key] = value
             elif key.startswith('x-image-meta-'):
-                _key = ensure_unicode(key[13:], incoming='utf-8')
+                _key = safe_decode(key[13:], incoming='utf-8')
                 meta[_key] = value
 
         for key in ['is_public', 'protected', 'deleted']:
@@ -154,7 +155,7 @@ class ImageManager(base.Manager):
             # trying to encode them
             for param, value in qp.iteritems():
                 if isinstance(value, basestring):
-                    qp[param] = utils.ensure_str(value)
+                    qp[param] = strutils.safe_encode(value)
 
             url = '/v1/images/detail?%s' % urllib.urlencode(qp)
             images = self._list(url, "images")

glanceclient/v1/shell.py

@@ -25,6 +25,7 @@ else:
 
 from glanceclient import exc
 from glanceclient.common import utils
+from glanceclient.openstack.common import strutils
 import glanceclient.v1.images
 
 #NOTE(bcwaldon): import deprecated cli functions
@@ -309,7 +310,7 @@ def do_image_delete(gc, args):
         try:
             if args.verbose:
                 print 'Requesting image delete for %s ...' % \
-                      utils.ensure_str(args_image),
+                      strutils.safe_encode(args_image),
 
             gc.images.delete(image)
 

glanceclient/v2/images.py

@@ -16,6 +16,7 @@
 import urllib
 
 from glanceclient.common import utils
+from glanceclient.openstack.common import strutils
 
 DEFAULT_PAGE_SIZE = 20
 
@@ -52,7 +53,7 @@ class Controller(object):
 
         for param, value in filters.iteritems():
             if isinstance(value, basestring):
-                filters[param] = utils.ensure_str(value)
+                filters[param] = strutils.safe_encode(value)
 
         url = '/v2/images?%s' % urllib.urlencode(filters)
 

openstack-common.conf

@@ -1,7 +1,9 @@
 [DEFAULT]
 
 # The list of modules to copy from openstack-common
+module=gettextutils
 module=importutils
+module=strutils
 
 # The base module to hold the copy of openstack.common
 base=glanceclient

tests/test_utils.py

@@ -53,24 +53,6 @@ class TestUtils(testtools.TestCase):
         self.assertEqual("1.4GB", utils.make_size_human_readable(1476395008))
         self.assertEqual("9.3MB", utils.make_size_human_readable(9761280))
 
-    def test_ensure_unicode(self):
-        ensure_unicode = utils.ensure_unicode
-        self.assertEqual(u'True', ensure_unicode(True))
-        self.assertEqual(u'ni\xf1o', ensure_unicode("ni\xc3\xb1o",
-                                                    incoming="utf-8"))
-        self.assertEqual(u"test", ensure_unicode("dGVzdA==",
-                                                 incoming='base64'))
-
-        self.assertEqual(u"strange", ensure_unicode('\x80strange',
-                                                    errors='ignore'))
-
-        self.assertEqual(u'\xc0', ensure_unicode('\xc0',
-                                                 incoming='iso-8859-1'))
-
-        # Forcing incoming to ascii so it falls back to utf-8
-        self.assertEqual(u'ni\xf1o', ensure_unicode('ni\xc3\xb1o',
-                                                    incoming='ascii'))
-
     def test_prettytable(self):
         class Struct:
             def __init__(self, **entries):
@@ -111,18 +93,3 @@ class TestUtils(testtools.TestCase):
 | Key      | Value |
 +----------+-------+
 ''')
-
-    def test_ensure_str(self):
-        ensure_str = utils.ensure_str
-        self.assertEqual("True", ensure_str(True))
-        self.assertEqual("ni\xc3\xb1o", ensure_str(u'ni\xf1o',
-                                                   encoding="utf-8"))
-        self.assertEqual("dGVzdA==\n", ensure_str("test",
-                                                  encoding='base64'))
-        self.assertEqual('ni\xf1o', ensure_str("ni\xc3\xb1o",
-                                               encoding="iso-8859-1",
-                                               incoming="utf-8"))
-
-        # Forcing incoming to ascii so it falls back to utf-8
-        self.assertEqual('ni\xc3\xb1o', ensure_str('ni\xc3\xb1o',
-                                                   incoming='ascii'))