Encode headers and params

BaseClient currently fails when non-ascii characters are passed in
paramas and headers. This patch ensures those params are correctly
encoded before the request is made.

The patch adds strutils from oslo and updates gettextutils.

Fixes bug: #1182824

Change-Id: I813d745d76700c77f1871b151a76ff8b72ea92d4

glance/common/client.py

@@ -43,6 +43,7 @@ except ImportError:
 from glance.common import auth
 from glance.common import exception, utils
 import glance.openstack.common.log as logging
+from glance.openstack.common import strutils
 
 LOG = logging.getLogger(__name__)
 
@@ -388,6 +389,10 @@ class BaseClient(object):
             for (key, value) in params.items():
                 if value is None:
                     del params[key]
+                    continue
+                if not isinstance(value, basestring):
+                    value = str(value)
+                params[key] = strutils.safe_encode(value)
             query = urllib.urlencode(params)
         else:
             query = None
@@ -397,6 +402,20 @@ class BaseClient(object):
         LOG.debug(log_msg, url.geturl())
         return url
 
+    def _encode_headers(self, headers):
+        """
+        Encodes headers.
+
+        Note: This should be used right before
+        sending anything out.
+
+        :param headers: Headers to encode
+        :returns: Dictionary with encoded headers'
+                  names and values
+        """
+        to_str = strutils.safe_encode
+        return dict([(to_str(h), to_str(v)) for h, v in headers.iteritems()])
+
     @handle_redirects
     def _do_request(self, method, url, body, headers):
         """
@@ -427,7 +446,7 @@ class BaseClient(object):
 
         try:
             connection_type = self.get_connection_type()
-            headers = headers or {}
+            headers = self._encode_headers(headers or {})
 
             if 'x-auth-token' not in headers and self.auth_tok:
                 headers['x-auth-token'] = self.auth_tok

glance/openstack/common/gettextutils.py

@@ -24,10 +24,27 @@ Usual usage in an openstack.common module:
 """
 
 import gettext
+import os
 
-
-t = gettext.translation('openstack-common', 'locale', fallback=True)
+_localedir = os.environ.get('glance'.upper() + '_LOCALEDIR')
+_t = gettext.translation('glance', localedir=_localedir, fallback=True)
 
 
 def _(msg):
-    return t.ugettext(msg)
+    return _t.ugettext(msg)
+
+
+def install(domain):
+    """Install a _() function using the given translation domain.
+
+    Given a translation domain, install a _() function using gettext's
+    install() function.
+
+    The main difference from gettext.install() is that we allow
+    overriding the default localedir (e.g. /usr/share/locale) using
+    a translation-domain-specific environment variable (e.g.
+    NOVA_LOCALEDIR).
+    """
+    gettext.install(domain,
+                    localedir=os.environ.get(domain.upper() + '_LOCALEDIR'),
+                    unicode=True)

glance/openstack/common/strutils.py

@@ -0,0 +1,150 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+System-level utilities and helper functions.
+"""
+
+import sys
+
+from glance.openstack.common.gettextutils import _
+
+
+TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
+FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
+
+
+def int_from_bool_as_string(subject):
+    """
+    Interpret a string as a boolean and return either 1 or 0.
+
+    Any string value in:
+
+        ('True', 'true', 'On', 'on', '1')
+
+    is interpreted as a boolean True.
+
+    Useful for JSON-decoded stuff and config file parsing
+    """
+    return bool_from_string(subject) and 1 or 0
+
+
+def bool_from_string(subject, strict=False):
+    """
+    Interpret a string as a boolean.
+
+    A case-insensitive match is performed such that strings matching 't',
+    'true', 'on', 'y', 'yes', or '1' are considered True and, when
+    `strict=False`, anything else is considered False.
+
+    Useful for JSON-decoded stuff and config file parsing.
+
+    If `strict=True`, unrecognized values, including None, will raise a
+    ValueError which is useful when parsing values passed in from an API call.
+    Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
+    """
+    if not isinstance(subject, basestring):
+        subject = str(subject)
+
+    lowered = subject.strip().lower()
+
+    if lowered in TRUE_STRINGS:
+        return True
+    elif lowered in FALSE_STRINGS:
+        return False
+    elif strict:
+        acceptable = ', '.join(
+            "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
+        msg = _("Unrecognized value '%(val)s', acceptable values are:"
+                " %(acceptable)s") % {'val': subject,
+                                      'acceptable': acceptable}
+        raise ValueError(msg)
+    else:
+        return False
+
+
+def safe_decode(text, incoming=None, errors='strict'):
+    """
+    Decodes incoming str using `incoming` if they're
+    not already unicode.
+
+    :param incoming: Text's current encoding
+    :param errors: Errors handling policy. See here for valid
+        values http://docs.python.org/2/library/codecs.html
+    :returns: text or a unicode `incoming` encoded
+                representation of it.
+    :raises TypeError: If text is not an isntance of basestring
+    """
+    if not isinstance(text, basestring):
+        raise TypeError("%s can't be decoded" % type(text))
+
+    if isinstance(text, unicode):
+        return text
+
+    if not incoming:
+        incoming = (sys.stdin.encoding or
+                    sys.getdefaultencoding())
+
+    try:
+        return text.decode(incoming, errors)
+    except UnicodeDecodeError:
+        # Note(flaper87) If we get here, it means that
+        # sys.stdin.encoding / sys.getdefaultencoding
+        # didn't return a suitable encoding to decode
+        # text. This happens mostly when global LANG
+        # var is not set correctly and there's no
+        # default encoding. In this case, most likely
+        # python will use ASCII or ANSI encoders as
+        # default encodings but they won't be capable
+        # of decoding non-ASCII characters.
+        #
+        # Also, UTF-8 is being used since it's an ASCII
+        # extension.
+        return text.decode('utf-8', errors)
+
+
+def safe_encode(text, incoming=None,
+                encoding='utf-8', errors='strict'):
+    """
+    Encodes incoming str/unicode using `encoding`. If
+    incoming is not specified, text is expected to
+    be encoded with current python's default encoding.
+    (`sys.getdefaultencoding`)
+
+    :param incoming: Text's current encoding
+    :param encoding: Expected encoding for text (Default UTF-8)
+    :param errors: Errors handling policy. See here for valid
+        values http://docs.python.org/2/library/codecs.html
+    :returns: text or a bytestring `encoding` encoded
+                representation of it.
+    :raises TypeError: If text is not an isntance of basestring
+    """
+    if not isinstance(text, basestring):
+        raise TypeError("%s can't be encoded" % type(text))
+
+    if not incoming:
+        incoming = (sys.stdin.encoding or
+                    sys.getdefaultencoding())
+
+    if isinstance(text, unicode):
+        return text.encode(encoding, errors)
+    elif text and encoding != incoming:
+        # Decode text before encoding it with `encoding`
+        text = safe_decode(text, incoming, errors)
+        return text.encode(encoding, errors)
+
+    return text

glance/tests/unit/common/test_client.py

@@ -0,0 +1,76 @@
+# Copyright 2013 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import httplib
+import StringIO
+
+import mox
+import testtools
+
+from glance.common import client
+from glance.tests import utils
+
+
+class TestClient(testtools.TestCase):
+
+    def setUp(self):
+        super(TestClient, self).setUp()
+        self.mock = mox.Mox()
+        self.mock.StubOutWithMock(httplib.HTTPConnection, 'request')
+        self.mock.StubOutWithMock(httplib.HTTPConnection, 'getresponse')
+
+        self.endpoint = 'example.com'
+        self.client = client.BaseClient(self.endpoint, port=9191,
+                                        auth_tok=u'abc123')
+
+    def tearDown(self):
+        super(TestClient, self).tearDown()
+        self.mock.UnsetStubs()
+
+    def test_http_encoding_headers(self):
+        httplib.HTTPConnection.request(
+            mox.IgnoreArg(),
+            mox.IgnoreArg(),
+            mox.IgnoreArg(),
+            mox.IgnoreArg())
+
+        # Lets fake the response
+        # returned by httplib
+        fake = utils.FakeHTTPResponse(data="Ok")
+        httplib.HTTPConnection.getresponse().AndReturn(fake)
+        self.mock.ReplayAll()
+
+        headers = {"test": u'ni\xf1o'}
+        resp = self.client.do_request('GET', '/v1/images/detail',
+                                      headers=headers)
+        self.assertEqual(resp, fake)
+
+    def test_http_encoding_params(self):
+        httplib.HTTPConnection.request(
+            mox.IgnoreArg(),
+            mox.IgnoreArg(),
+            mox.IgnoreArg(),
+            mox.IgnoreArg())
+
+        # Lets fake the response
+        # returned by httplib
+        fake = utils.FakeHTTPResponse(data="Ok")
+        httplib.HTTPConnection.getresponse().AndReturn(fake)
+        self.mock.ReplayAll()
+
+        params = {"test": u'ni\xf1o'}
+        resp = self.client.do_request('GET', '/v1/images/detail',
+                                      params=params)
+        self.assertEqual(resp, fake)

openstack-common.conf

@@ -10,6 +10,7 @@ module=log
 module=notifier
 module=policy
 module=setup
+module=strutils
 module=timeutils
 module=uuidutils
 module=version