Replace utils.ensure_(str|unicode) with strutils.safe(decode|encode)

Glanceclient implemented both functions before they landed into oslo.
Since both functions are already in oslo, it is now possible to pull
them in.

There's a small difference between glance's implementation and oslo's,
that is the later does not convert non-str objects - int, bool - to str
before trying to decode / encode them. This patch takes care of that
where necessary, more precisely, while encoding headers before doing a
new request.

Fixes bug: #1172253

Change-Id: I9a0dca31140bae28d8ec6aede515c5bb852b701b
This commit is contained in:
Flaper Fesp 2013-05-22 11:31:25 +02:00
parent 7daa976d14
commit 7818387d4a
10 changed files with 224 additions and 122 deletions

View File

@ -36,6 +36,7 @@ import OpenSSL
from glanceclient import exc
from glanceclient.common import utils
from glanceclient.openstack.common import strutils
try:
from eventlet import patcher
@ -130,7 +131,7 @@ class HTTPClient(object):
curl.append('-d \'%s\'' % kwargs['body'])
curl.append('%s%s' % (self.endpoint, url))
LOG.debug(utils.ensure_str(' '.join(curl)))
LOG.debug(strutils.safe_encode(' '.join(curl)))
@staticmethod
def log_http_response(resp, body=None):
@ -140,7 +141,7 @@ class HTTPClient(object):
dump.append('')
if body:
dump.extend([body, ''])
LOG.debug(utils.ensure_str('\n'.join(dump)))
LOG.debug(strutils.safe_encode('\n'.join(dump)))
@staticmethod
def encode_headers(headers):
@ -154,7 +155,7 @@ class HTTPClient(object):
:returns: Dictionary with encoded headers'
names and values
"""
to_str = utils.ensure_str
to_str = strutils.safe_encode
return dict([(to_str(h), to_str(v)) for h, v in headers.iteritems()])
def _http_request(self, url, method, **kwargs):
@ -182,7 +183,7 @@ class HTTPClient(object):
conn_url = posixpath.normpath('%s/%s' % (self.endpoint_path, url))
# Note(flaper87): Ditto, headers / url
# encoding to make httplib happy.
conn_url = utils.ensure_str(conn_url)
conn_url = strutils.safe_encode(conn_url)
if kwargs['headers'].get('Transfer-Encoding') == 'chunked':
conn.putrequest(method, conn_url)
for header, value in kwargs['headers'].items():

View File

@ -23,6 +23,7 @@ import prettytable
from glanceclient import exc
from glanceclient.openstack.common import importutils
from glanceclient.openstack.common import strutils
# Decorator for cli-args
@ -54,14 +55,14 @@ def print_list(objs, fields, formatters={}):
row.append(data)
pt.add_row(row)
print ensure_str(pt.get_string())
print strutils.safe_encode(pt.get_string())
def print_dict(d):
pt = prettytable.PrettyTable(['Property', 'Value'], caching=False)
pt.align = 'l'
[pt.add_row(list(r)) for r in d.iteritems()]
print ensure_str(pt.get_string(sortby='Property'))
print strutils.safe_encode(pt.get_string(sortby='Property'))
def find_resource(manager, name_or_id):
@ -75,7 +76,7 @@ def find_resource(manager, name_or_id):
# now try to get entity as uuid
try:
uuid.UUID(ensure_str(name_or_id))
uuid.UUID(strutils.safe_encode(name_or_id))
return manager.get(name_or_id)
except (ValueError, exc.NotFound):
pass
@ -137,7 +138,7 @@ def import_versioned_module(version, submodule=None):
def exit(msg=''):
if msg:
print >> sys.stderr, ensure_str(msg)
print >> sys.stderr, strutils.safe_encode(msg)
sys.exit(1)
@ -192,79 +193,6 @@ def make_size_human_readable(size):
return '%s%s' % (stripped, suffix[index])
def ensure_unicode(text, incoming=None, errors='strict'):
"""
Decodes incoming objects using `incoming` if they're
not already unicode.
:param incoming: Text's current encoding
:param errors: Errors handling policy.
:returns: text or a unicode `incoming` encoded
representation of it.
"""
if isinstance(text, unicode):
return text
if not incoming:
incoming = sys.stdin.encoding or \
sys.getdefaultencoding()
# Calling `str` in case text is a non str
# object.
text = str(text)
try:
return text.decode(incoming, errors)
except UnicodeDecodeError:
# Note(flaper87) If we get here, it means that
# sys.stdin.encoding / sys.getdefaultencoding
# didn't return a suitable encoding to decode
# text. This happens mostly when global LANG
# var is not set correctly and there's no
# default encoding. In this case, most likely
# python will use ASCII or ANSI encoders as
# default encodings but they won't be capable
# of decoding non-ASCII characters.
#
# Also, UTF-8 is being used since it's an ASCII
# extension.
return text.decode('utf-8', errors)
def ensure_str(text, incoming=None,
encoding='utf-8', errors='strict'):
"""
Encodes incoming objects using `encoding`. If
incoming is not specified, text is expected to
be encoded with current python's default encoding.
(`sys.getdefaultencoding`)
:param incoming: Text's current encoding
:param encoding: Expected encoding for text (Default UTF-8)
:param errors: Errors handling policy.
:returns: text or a bytestring `encoding` encoded
representation of it.
"""
if not incoming:
incoming = sys.stdin.encoding or \
sys.getdefaultencoding()
if not isinstance(text, basestring):
# try to convert `text` to string
# This allows this method for receiving
# objs that can be converted to string
text = str(text)
if isinstance(text, unicode):
return text.encode(encoding, errors)
elif text and encoding != incoming:
# Decode text before encoding it with `encoding`
text = ensure_unicode(text, incoming, errors)
return text.encode(encoding, errors)
return text
def getsockopt(self, *args, **kwargs):
"""
A function which allows us to monkey patch eventlet's

View File

@ -0,0 +1,50 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
gettext for openstack-common modules.
Usual usage in an openstack.common module:
from glanceclient.openstack.common.gettextutils import _
"""
import gettext
import os
_localedir = os.environ.get('glanceclient'.upper() + '_LOCALEDIR')
_t = gettext.translation('glanceclient', localedir=_localedir, fallback=True)
def _(msg):
return _t.ugettext(msg)
def install(domain):
"""Install a _() function using the given translation domain.
Given a translation domain, install a _() function using gettext's
install() function.
The main difference from gettext.install() is that we allow
overriding the default localedir (e.g. /usr/share/locale) using
a translation-domain-specific environment variable (e.g.
NOVA_LOCALEDIR).
"""
gettext.install(domain,
localedir=os.environ.get(domain.upper() + '_LOCALEDIR'),
unicode=True)

View File

@ -0,0 +1,150 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
System-level utilities and helper functions.
"""
import sys
from glanceclient.openstack.common.gettextutils import _
TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
def int_from_bool_as_string(subject):
"""
Interpret a string as a boolean and return either 1 or 0.
Any string value in:
('True', 'true', 'On', 'on', '1')
is interpreted as a boolean True.
Useful for JSON-decoded stuff and config file parsing
"""
return bool_from_string(subject) and 1 or 0
def bool_from_string(subject, strict=False):
"""
Interpret a string as a boolean.
A case-insensitive match is performed such that strings matching 't',
'true', 'on', 'y', 'yes', or '1' are considered True and, when
`strict=False`, anything else is considered False.
Useful for JSON-decoded stuff and config file parsing.
If `strict=True`, unrecognized values, including None, will raise a
ValueError which is useful when parsing values passed in from an API call.
Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
"""
if not isinstance(subject, basestring):
subject = str(subject)
lowered = subject.strip().lower()
if lowered in TRUE_STRINGS:
return True
elif lowered in FALSE_STRINGS:
return False
elif strict:
acceptable = ', '.join(
"'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
msg = _("Unrecognized value '%(val)s', acceptable values are:"
" %(acceptable)s") % {'val': subject,
'acceptable': acceptable}
raise ValueError(msg)
else:
return False
def safe_decode(text, incoming=None, errors='strict'):
"""
Decodes incoming str using `incoming` if they're
not already unicode.
:param incoming: Text's current encoding
:param errors: Errors handling policy. See here for valid
values http://docs.python.org/2/library/codecs.html
:returns: text or a unicode `incoming` encoded
representation of it.
:raises TypeError: If text is not an isntance of basestring
"""
if not isinstance(text, basestring):
raise TypeError("%s can't be decoded" % type(text))
if isinstance(text, unicode):
return text
if not incoming:
incoming = (sys.stdin.encoding or
sys.getdefaultencoding())
try:
return text.decode(incoming, errors)
except UnicodeDecodeError:
# Note(flaper87) If we get here, it means that
# sys.stdin.encoding / sys.getdefaultencoding
# didn't return a suitable encoding to decode
# text. This happens mostly when global LANG
# var is not set correctly and there's no
# default encoding. In this case, most likely
# python will use ASCII or ANSI encoders as
# default encodings but they won't be capable
# of decoding non-ASCII characters.
#
# Also, UTF-8 is being used since it's an ASCII
# extension.
return text.decode('utf-8', errors)
def safe_encode(text, incoming=None,
encoding='utf-8', errors='strict'):
"""
Encodes incoming str/unicode using `encoding`. If
incoming is not specified, text is expected to
be encoded with current python's default encoding.
(`sys.getdefaultencoding`)
:param incoming: Text's current encoding
:param encoding: Expected encoding for text (Default UTF-8)
:param errors: Errors handling policy. See here for valid
values http://docs.python.org/2/library/codecs.html
:returns: text or a bytestring `encoding` encoded
representation of it.
:raises TypeError: If text is not an isntance of basestring
"""
if not isinstance(text, basestring):
raise TypeError("%s can't be encoded" % type(text))
if not incoming:
incoming = (sys.stdin.encoding or
sys.getdefaultencoding())
if isinstance(text, unicode):
return text.encode(encoding, errors)
elif text and encoding != incoming:
# Decode text before encoding it with `encoding`
text = safe_decode(text, incoming, errors)
return text.encode(encoding, errors)
return text

View File

@ -27,6 +27,7 @@ from keystoneclient.v2_0 import client as ksclient
import glanceclient
from glanceclient import exc
from glanceclient.common import utils
from glanceclient.openstack.common import strutils
class OpenStackImagesShell(object):
@ -466,7 +467,7 @@ class HelpFormatter(argparse.HelpFormatter):
def main():
try:
OpenStackImagesShell().main(map(utils.ensure_unicode, sys.argv[1:]))
OpenStackImagesShell().main(map(strutils.safe_decode, sys.argv[1:]))
except KeyboardInterrupt:
print >> sys.stderr, '... terminating glance client'
sys.exit(1)

View File

@ -21,6 +21,7 @@ import urllib
from glanceclient.common import base
from glanceclient.common import utils
from glanceclient.openstack.common import strutils
UPDATE_PARAMS = ('name', 'disk_format', 'container_format', 'min_disk',
'min_ram', 'owner', 'size', 'is_public', 'protected',
@ -58,14 +59,14 @@ class ImageManager(base.Manager):
def _image_meta_from_headers(self, headers):
meta = {'properties': {}}
ensure_unicode = utils.ensure_unicode
safe_decode = strutils.safe_decode
for key, value in headers.iteritems():
value = ensure_unicode(value, incoming='utf-8')
value = safe_decode(value, incoming='utf-8')
if key.startswith('x-image-meta-property-'):
_key = ensure_unicode(key[22:], incoming='utf-8')
_key = safe_decode(key[22:], incoming='utf-8')
meta['properties'][_key] = value
elif key.startswith('x-image-meta-'):
_key = ensure_unicode(key[13:], incoming='utf-8')
_key = safe_decode(key[13:], incoming='utf-8')
meta[_key] = value
for key in ['is_public', 'protected', 'deleted']:
@ -154,7 +155,7 @@ class ImageManager(base.Manager):
# trying to encode them
for param, value in qp.iteritems():
if isinstance(value, basestring):
qp[param] = utils.ensure_str(value)
qp[param] = strutils.safe_encode(value)
url = '/v1/images/detail?%s' % urllib.urlencode(qp)
images = self._list(url, "images")

View File

@ -25,6 +25,7 @@ else:
from glanceclient import exc
from glanceclient.common import utils
from glanceclient.openstack.common import strutils
import glanceclient.v1.images
#NOTE(bcwaldon): import deprecated cli functions
@ -309,7 +310,7 @@ def do_image_delete(gc, args):
try:
if args.verbose:
print 'Requesting image delete for %s ...' % \
utils.ensure_str(args_image),
strutils.safe_encode(args_image),
gc.images.delete(image)

View File

@ -16,6 +16,7 @@
import urllib
from glanceclient.common import utils
from glanceclient.openstack.common import strutils
DEFAULT_PAGE_SIZE = 20
@ -52,7 +53,7 @@ class Controller(object):
for param, value in filters.iteritems():
if isinstance(value, basestring):
filters[param] = utils.ensure_str(value)
filters[param] = strutils.safe_encode(value)
url = '/v2/images?%s' % urllib.urlencode(filters)

View File

@ -1,7 +1,9 @@
[DEFAULT]
# The list of modules to copy from openstack-common
module=gettextutils
module=importutils
module=strutils
# The base module to hold the copy of openstack.common
base=glanceclient

View File

@ -53,24 +53,6 @@ class TestUtils(testtools.TestCase):
self.assertEqual("1.4GB", utils.make_size_human_readable(1476395008))
self.assertEqual("9.3MB", utils.make_size_human_readable(9761280))
def test_ensure_unicode(self):
ensure_unicode = utils.ensure_unicode
self.assertEqual(u'True', ensure_unicode(True))
self.assertEqual(u'ni\xf1o', ensure_unicode("ni\xc3\xb1o",
incoming="utf-8"))
self.assertEqual(u"test", ensure_unicode("dGVzdA==",
incoming='base64'))
self.assertEqual(u"strange", ensure_unicode('\x80strange',
errors='ignore'))
self.assertEqual(u'\xc0', ensure_unicode('\xc0',
incoming='iso-8859-1'))
# Forcing incoming to ascii so it falls back to utf-8
self.assertEqual(u'ni\xf1o', ensure_unicode('ni\xc3\xb1o',
incoming='ascii'))
def test_prettytable(self):
class Struct:
def __init__(self, **entries):
@ -111,18 +93,3 @@ class TestUtils(testtools.TestCase):
| Key | Value |
+----------+-------+
''')
def test_ensure_str(self):
ensure_str = utils.ensure_str
self.assertEqual("True", ensure_str(True))
self.assertEqual("ni\xc3\xb1o", ensure_str(u'ni\xf1o',
encoding="utf-8"))
self.assertEqual("dGVzdA==\n", ensure_str("test",
encoding='base64'))
self.assertEqual('ni\xf1o', ensure_str("ni\xc3\xb1o",
encoding="iso-8859-1",
incoming="utf-8"))
# Forcing incoming to ascii so it falls back to utf-8
self.assertEqual('ni\xc3\xb1o', ensure_str('ni\xc3\xb1o',
incoming='ascii'))