From 7818387d4ac4c4a20899fc4470da86c5bcabf183 Mon Sep 17 00:00:00 2001 From: Flaper Fesp Date: Wed, 22 May 2013 11:31:25 +0200 Subject: [PATCH] Replace utils.ensure_(str|unicode) with strutils.safe(decode|encode) Glanceclient implemented both functions before they landed into oslo. Since both functions are already in oslo, it is now possible to pull them in. There's a small difference between glance's implementation and oslo's, that is the later does not convert non-str objects - int, bool - to str before trying to decode / encode them. This patch takes care of that where necessary, more precisely, while encoding headers before doing a new request. Fixes bug: #1172253 Change-Id: I9a0dca31140bae28d8ec6aede515c5bb852b701b --- glanceclient/common/http.py | 9 +- glanceclient/common/utils.py | 82 +--------- glanceclient/openstack/common/gettextutils.py | 50 ++++++ glanceclient/openstack/common/strutils.py | 150 ++++++++++++++++++ glanceclient/shell.py | 3 +- glanceclient/v1/images.py | 11 +- glanceclient/v1/shell.py | 3 +- glanceclient/v2/images.py | 3 +- openstack-common.conf | 2 + tests/test_utils.py | 33 ---- 10 files changed, 224 insertions(+), 122 deletions(-) create mode 100644 glanceclient/openstack/common/gettextutils.py create mode 100644 glanceclient/openstack/common/strutils.py diff --git a/glanceclient/common/http.py b/glanceclient/common/http.py index b831cb78..f6fb4d8c 100644 --- a/glanceclient/common/http.py +++ b/glanceclient/common/http.py @@ -36,6 +36,7 @@ import OpenSSL from glanceclient import exc from glanceclient.common import utils +from glanceclient.openstack.common import strutils try: from eventlet import patcher @@ -130,7 +131,7 @@ class HTTPClient(object): curl.append('-d \'%s\'' % kwargs['body']) curl.append('%s%s' % (self.endpoint, url)) - LOG.debug(utils.ensure_str(' '.join(curl))) + LOG.debug(strutils.safe_encode(' '.join(curl))) @staticmethod def log_http_response(resp, body=None): @@ -140,7 +141,7 @@ class HTTPClient(object): dump.append('') if body: dump.extend([body, '']) - LOG.debug(utils.ensure_str('\n'.join(dump))) + LOG.debug(strutils.safe_encode('\n'.join(dump))) @staticmethod def encode_headers(headers): @@ -154,7 +155,7 @@ class HTTPClient(object): :returns: Dictionary with encoded headers' names and values """ - to_str = utils.ensure_str + to_str = strutils.safe_encode return dict([(to_str(h), to_str(v)) for h, v in headers.iteritems()]) def _http_request(self, url, method, **kwargs): @@ -182,7 +183,7 @@ class HTTPClient(object): conn_url = posixpath.normpath('%s/%s' % (self.endpoint_path, url)) # Note(flaper87): Ditto, headers / url # encoding to make httplib happy. - conn_url = utils.ensure_str(conn_url) + conn_url = strutils.safe_encode(conn_url) if kwargs['headers'].get('Transfer-Encoding') == 'chunked': conn.putrequest(method, conn_url) for header, value in kwargs['headers'].items(): diff --git a/glanceclient/common/utils.py b/glanceclient/common/utils.py index 30dcd58b..49976ac6 100644 --- a/glanceclient/common/utils.py +++ b/glanceclient/common/utils.py @@ -23,6 +23,7 @@ import prettytable from glanceclient import exc from glanceclient.openstack.common import importutils +from glanceclient.openstack.common import strutils # Decorator for cli-args @@ -54,14 +55,14 @@ def print_list(objs, fields, formatters={}): row.append(data) pt.add_row(row) - print ensure_str(pt.get_string()) + print strutils.safe_encode(pt.get_string()) def print_dict(d): pt = prettytable.PrettyTable(['Property', 'Value'], caching=False) pt.align = 'l' [pt.add_row(list(r)) for r in d.iteritems()] - print ensure_str(pt.get_string(sortby='Property')) + print strutils.safe_encode(pt.get_string(sortby='Property')) def find_resource(manager, name_or_id): @@ -75,7 +76,7 @@ def find_resource(manager, name_or_id): # now try to get entity as uuid try: - uuid.UUID(ensure_str(name_or_id)) + uuid.UUID(strutils.safe_encode(name_or_id)) return manager.get(name_or_id) except (ValueError, exc.NotFound): pass @@ -137,7 +138,7 @@ def import_versioned_module(version, submodule=None): def exit(msg=''): if msg: - print >> sys.stderr, ensure_str(msg) + print >> sys.stderr, strutils.safe_encode(msg) sys.exit(1) @@ -192,79 +193,6 @@ def make_size_human_readable(size): return '%s%s' % (stripped, suffix[index]) -def ensure_unicode(text, incoming=None, errors='strict'): - """ - Decodes incoming objects using `incoming` if they're - not already unicode. - - :param incoming: Text's current encoding - :param errors: Errors handling policy. - :returns: text or a unicode `incoming` encoded - representation of it. - """ - if isinstance(text, unicode): - return text - - if not incoming: - incoming = sys.stdin.encoding or \ - sys.getdefaultencoding() - - # Calling `str` in case text is a non str - # object. - text = str(text) - try: - return text.decode(incoming, errors) - except UnicodeDecodeError: - # Note(flaper87) If we get here, it means that - # sys.stdin.encoding / sys.getdefaultencoding - # didn't return a suitable encoding to decode - # text. This happens mostly when global LANG - # var is not set correctly and there's no - # default encoding. In this case, most likely - # python will use ASCII or ANSI encoders as - # default encodings but they won't be capable - # of decoding non-ASCII characters. - # - # Also, UTF-8 is being used since it's an ASCII - # extension. - return text.decode('utf-8', errors) - - -def ensure_str(text, incoming=None, - encoding='utf-8', errors='strict'): - """ - Encodes incoming objects using `encoding`. If - incoming is not specified, text is expected to - be encoded with current python's default encoding. - (`sys.getdefaultencoding`) - - :param incoming: Text's current encoding - :param encoding: Expected encoding for text (Default UTF-8) - :param errors: Errors handling policy. - :returns: text or a bytestring `encoding` encoded - representation of it. - """ - - if not incoming: - incoming = sys.stdin.encoding or \ - sys.getdefaultencoding() - - if not isinstance(text, basestring): - # try to convert `text` to string - # This allows this method for receiving - # objs that can be converted to string - text = str(text) - - if isinstance(text, unicode): - return text.encode(encoding, errors) - elif text and encoding != incoming: - # Decode text before encoding it with `encoding` - text = ensure_unicode(text, incoming, errors) - return text.encode(encoding, errors) - - return text - - def getsockopt(self, *args, **kwargs): """ A function which allows us to monkey patch eventlet's diff --git a/glanceclient/openstack/common/gettextutils.py b/glanceclient/openstack/common/gettextutils.py new file mode 100644 index 00000000..edb0b30d --- /dev/null +++ b/glanceclient/openstack/common/gettextutils.py @@ -0,0 +1,50 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +gettext for openstack-common modules. + +Usual usage in an openstack.common module: + + from glanceclient.openstack.common.gettextutils import _ +""" + +import gettext +import os + +_localedir = os.environ.get('glanceclient'.upper() + '_LOCALEDIR') +_t = gettext.translation('glanceclient', localedir=_localedir, fallback=True) + + +def _(msg): + return _t.ugettext(msg) + + +def install(domain): + """Install a _() function using the given translation domain. + + Given a translation domain, install a _() function using gettext's + install() function. + + The main difference from gettext.install() is that we allow + overriding the default localedir (e.g. /usr/share/locale) using + a translation-domain-specific environment variable (e.g. + NOVA_LOCALEDIR). + """ + gettext.install(domain, + localedir=os.environ.get(domain.upper() + '_LOCALEDIR'), + unicode=True) diff --git a/glanceclient/openstack/common/strutils.py b/glanceclient/openstack/common/strutils.py new file mode 100644 index 00000000..a5c5448b --- /dev/null +++ b/glanceclient/openstack/common/strutils.py @@ -0,0 +1,150 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +System-level utilities and helper functions. +""" + +import sys + +from glanceclient.openstack.common.gettextutils import _ + + +TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes') +FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no') + + +def int_from_bool_as_string(subject): + """ + Interpret a string as a boolean and return either 1 or 0. + + Any string value in: + + ('True', 'true', 'On', 'on', '1') + + is interpreted as a boolean True. + + Useful for JSON-decoded stuff and config file parsing + """ + return bool_from_string(subject) and 1 or 0 + + +def bool_from_string(subject, strict=False): + """ + Interpret a string as a boolean. + + A case-insensitive match is performed such that strings matching 't', + 'true', 'on', 'y', 'yes', or '1' are considered True and, when + `strict=False`, anything else is considered False. + + Useful for JSON-decoded stuff and config file parsing. + + If `strict=True`, unrecognized values, including None, will raise a + ValueError which is useful when parsing values passed in from an API call. + Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. + """ + if not isinstance(subject, basestring): + subject = str(subject) + + lowered = subject.strip().lower() + + if lowered in TRUE_STRINGS: + return True + elif lowered in FALSE_STRINGS: + return False + elif strict: + acceptable = ', '.join( + "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS)) + msg = _("Unrecognized value '%(val)s', acceptable values are:" + " %(acceptable)s") % {'val': subject, + 'acceptable': acceptable} + raise ValueError(msg) + else: + return False + + +def safe_decode(text, incoming=None, errors='strict'): + """ + Decodes incoming str using `incoming` if they're + not already unicode. + + :param incoming: Text's current encoding + :param errors: Errors handling policy. See here for valid + values http://docs.python.org/2/library/codecs.html + :returns: text or a unicode `incoming` encoded + representation of it. + :raises TypeError: If text is not an isntance of basestring + """ + if not isinstance(text, basestring): + raise TypeError("%s can't be decoded" % type(text)) + + if isinstance(text, unicode): + return text + + if not incoming: + incoming = (sys.stdin.encoding or + sys.getdefaultencoding()) + + try: + return text.decode(incoming, errors) + except UnicodeDecodeError: + # Note(flaper87) If we get here, it means that + # sys.stdin.encoding / sys.getdefaultencoding + # didn't return a suitable encoding to decode + # text. This happens mostly when global LANG + # var is not set correctly and there's no + # default encoding. In this case, most likely + # python will use ASCII or ANSI encoders as + # default encodings but they won't be capable + # of decoding non-ASCII characters. + # + # Also, UTF-8 is being used since it's an ASCII + # extension. + return text.decode('utf-8', errors) + + +def safe_encode(text, incoming=None, + encoding='utf-8', errors='strict'): + """ + Encodes incoming str/unicode using `encoding`. If + incoming is not specified, text is expected to + be encoded with current python's default encoding. + (`sys.getdefaultencoding`) + + :param incoming: Text's current encoding + :param encoding: Expected encoding for text (Default UTF-8) + :param errors: Errors handling policy. See here for valid + values http://docs.python.org/2/library/codecs.html + :returns: text or a bytestring `encoding` encoded + representation of it. + :raises TypeError: If text is not an isntance of basestring + """ + if not isinstance(text, basestring): + raise TypeError("%s can't be encoded" % type(text)) + + if not incoming: + incoming = (sys.stdin.encoding or + sys.getdefaultencoding()) + + if isinstance(text, unicode): + return text.encode(encoding, errors) + elif text and encoding != incoming: + # Decode text before encoding it with `encoding` + text = safe_decode(text, incoming, errors) + return text.encode(encoding, errors) + + return text diff --git a/glanceclient/shell.py b/glanceclient/shell.py index 417c7847..58f7686d 100644 --- a/glanceclient/shell.py +++ b/glanceclient/shell.py @@ -27,6 +27,7 @@ from keystoneclient.v2_0 import client as ksclient import glanceclient from glanceclient import exc from glanceclient.common import utils +from glanceclient.openstack.common import strutils class OpenStackImagesShell(object): @@ -466,7 +467,7 @@ class HelpFormatter(argparse.HelpFormatter): def main(): try: - OpenStackImagesShell().main(map(utils.ensure_unicode, sys.argv[1:])) + OpenStackImagesShell().main(map(strutils.safe_decode, sys.argv[1:])) except KeyboardInterrupt: print >> sys.stderr, '... terminating glance client' sys.exit(1) diff --git a/glanceclient/v1/images.py b/glanceclient/v1/images.py index 152b8c73..05d687ce 100644 --- a/glanceclient/v1/images.py +++ b/glanceclient/v1/images.py @@ -21,6 +21,7 @@ import urllib from glanceclient.common import base from glanceclient.common import utils +from glanceclient.openstack.common import strutils UPDATE_PARAMS = ('name', 'disk_format', 'container_format', 'min_disk', 'min_ram', 'owner', 'size', 'is_public', 'protected', @@ -58,14 +59,14 @@ class ImageManager(base.Manager): def _image_meta_from_headers(self, headers): meta = {'properties': {}} - ensure_unicode = utils.ensure_unicode + safe_decode = strutils.safe_decode for key, value in headers.iteritems(): - value = ensure_unicode(value, incoming='utf-8') + value = safe_decode(value, incoming='utf-8') if key.startswith('x-image-meta-property-'): - _key = ensure_unicode(key[22:], incoming='utf-8') + _key = safe_decode(key[22:], incoming='utf-8') meta['properties'][_key] = value elif key.startswith('x-image-meta-'): - _key = ensure_unicode(key[13:], incoming='utf-8') + _key = safe_decode(key[13:], incoming='utf-8') meta[_key] = value for key in ['is_public', 'protected', 'deleted']: @@ -154,7 +155,7 @@ class ImageManager(base.Manager): # trying to encode them for param, value in qp.iteritems(): if isinstance(value, basestring): - qp[param] = utils.ensure_str(value) + qp[param] = strutils.safe_encode(value) url = '/v1/images/detail?%s' % urllib.urlencode(qp) images = self._list(url, "images") diff --git a/glanceclient/v1/shell.py b/glanceclient/v1/shell.py index 31d9f9fa..d1a1acee 100644 --- a/glanceclient/v1/shell.py +++ b/glanceclient/v1/shell.py @@ -25,6 +25,7 @@ else: from glanceclient import exc from glanceclient.common import utils +from glanceclient.openstack.common import strutils import glanceclient.v1.images #NOTE(bcwaldon): import deprecated cli functions @@ -309,7 +310,7 @@ def do_image_delete(gc, args): try: if args.verbose: print 'Requesting image delete for %s ...' % \ - utils.ensure_str(args_image), + strutils.safe_encode(args_image), gc.images.delete(image) diff --git a/glanceclient/v2/images.py b/glanceclient/v2/images.py index 0b9cf451..6060eef8 100644 --- a/glanceclient/v2/images.py +++ b/glanceclient/v2/images.py @@ -16,6 +16,7 @@ import urllib from glanceclient.common import utils +from glanceclient.openstack.common import strutils DEFAULT_PAGE_SIZE = 20 @@ -52,7 +53,7 @@ class Controller(object): for param, value in filters.iteritems(): if isinstance(value, basestring): - filters[param] = utils.ensure_str(value) + filters[param] = strutils.safe_encode(value) url = '/v2/images?%s' % urllib.urlencode(filters) diff --git a/openstack-common.conf b/openstack-common.conf index c0a8057c..e2ee4c64 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,7 +1,9 @@ [DEFAULT] # The list of modules to copy from openstack-common +module=gettextutils module=importutils +module=strutils # The base module to hold the copy of openstack.common base=glanceclient diff --git a/tests/test_utils.py b/tests/test_utils.py index df16acf5..138ce879 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -53,24 +53,6 @@ class TestUtils(testtools.TestCase): self.assertEqual("1.4GB", utils.make_size_human_readable(1476395008)) self.assertEqual("9.3MB", utils.make_size_human_readable(9761280)) - def test_ensure_unicode(self): - ensure_unicode = utils.ensure_unicode - self.assertEqual(u'True', ensure_unicode(True)) - self.assertEqual(u'ni\xf1o', ensure_unicode("ni\xc3\xb1o", - incoming="utf-8")) - self.assertEqual(u"test", ensure_unicode("dGVzdA==", - incoming='base64')) - - self.assertEqual(u"strange", ensure_unicode('\x80strange', - errors='ignore')) - - self.assertEqual(u'\xc0', ensure_unicode('\xc0', - incoming='iso-8859-1')) - - # Forcing incoming to ascii so it falls back to utf-8 - self.assertEqual(u'ni\xf1o', ensure_unicode('ni\xc3\xb1o', - incoming='ascii')) - def test_prettytable(self): class Struct: def __init__(self, **entries): @@ -111,18 +93,3 @@ class TestUtils(testtools.TestCase): | Key | Value | +----------+-------+ ''') - - def test_ensure_str(self): - ensure_str = utils.ensure_str - self.assertEqual("True", ensure_str(True)) - self.assertEqual("ni\xc3\xb1o", ensure_str(u'ni\xf1o', - encoding="utf-8")) - self.assertEqual("dGVzdA==\n", ensure_str("test", - encoding='base64')) - self.assertEqual('ni\xf1o', ensure_str("ni\xc3\xb1o", - encoding="iso-8859-1", - incoming="utf-8")) - - # Forcing incoming to ascii so it falls back to utf-8 - self.assertEqual('ni\xc3\xb1o', ensure_str('ni\xc3\xb1o', - incoming='ascii'))