Force utf-8 encode of HTTPConnection params

This patch forces swiftclient to encode to utf-8
all url and headers arguments, to avoid the
UnicodeDecodeError which is raised by '\r\n'.join([])
invoked in htplib.py.

Currently the affected projects are Horizon(upload file
with unicode name) and swiftclient CLI('swift post' with
unicode filename as header)

This is also a follow-up of this review:
    https://review.openstack.org/#/c/14216/

I'd still want to hear what the Swift core devs
think of it. Is it better to create a new
AutoEncodingHTTPConnection? Or to handle the connection
creation and make sure there are no unicode and utf-8
string at the same time. If these unicode checks have to
be added in the calling code(Dashboard, CLI), there are
so many places to be added, and also in all new commands
that might be exposed from the API.

Fixes bug 1008940

Change-Id: Ice2aa29024429d3e6f569a88d5cf8b4202537827

swiftclient/client.py

@@ -20,6 +20,7 @@ Cloud Files client library used internally
 import socket
 import os
 import logging
+from functools import wraps
 
 from urllib import quote as _quote
 from urlparse import urlparse, urlunparse
@@ -81,9 +82,17 @@ def quote(value, safe='/'):
     """
     Patched version of urllib.quote that encodes utf8 strings before quoting
     """
+    value = encode_utf8(value)
+    if isinstance(value, str):
+        return _quote(value, safe)
+    else:
+        return value
+
+
+def encode_utf8(value):
     if isinstance(value, unicode):
         value = value.encode('utf8')
-    return _quote(value, safe)
+    return value
 
 
 # look for a real json parser first
@@ -161,6 +170,7 @@ def http_connection(url, proxy=None):
     :returns: tuple of (parsed url, connection object)
     :raises ClientException: Unable to handle protocol scheme
     """
+    url = encode_utf8(url)
     parsed = urlparse(url)
     proxy_parsed = urlparse(proxy) if proxy else None
     if parsed.scheme == 'http':
@@ -170,6 +180,25 @@ def http_connection(url, proxy=None):
     else:
         raise ClientException('Cannot handle protocol scheme %s for url %s' %
                               (parsed.scheme, repr(url)))
+
+    def putheader_wrapper(func):
+
+        @wraps(func)
+        def putheader_escaped(key, value):
+            func(encode_utf8(key), encode_utf8(value))
+        return putheader_escaped
+    conn.putheader = putheader_wrapper(conn.putheader)
+
+    def request_wrapper(func):
+
+        @wraps(func)
+        def request_escaped(method, url, body=None, headers=None):
+            url = encode_utf8(url)
+            if body:
+                body = encode_utf8(body)
+            func(method, url, body=body, headers=headers or {})
+        return request_escaped
+    conn.request = request_wrapper(conn.request)
     if proxy:
         conn._set_tunnel(parsed.hostname, parsed.port)
     return parsed, conn

tests/test_swiftclient.py

@@ -15,6 +15,7 @@
 
 # TODO: More tests
 import socket
+import StringIO
 import unittest
 from urlparse import urlparse
 
@@ -121,6 +122,24 @@ class MockHttpTest(unittest.TestCase):
         reload(c)
 
 
+class MockHttpResponse():
+    def __init__(self):
+        self.status = 200
+        self.buffer = []
+
+    def read(self):
+        return ""
+
+    def getheader(self, name, default):
+        return ""
+
+    def fake_response(self):
+        return MockHttpResponse()
+
+    def fake_send(self, msg):
+        self.buffer.append(msg)
+
+
 class TestHttpHelpers(MockHttpTest):
 
     def test_quote(self):
@@ -360,6 +379,26 @@ class TestPutObject(MockHttpTest):
         value = c.put_object(*args)
         self.assertTrue(isinstance(value, basestring))
 
+    def test_unicode_ok(self):
+        conn = c.http_connection(u'http://www.test.com/')
+        file = StringIO.StringIO(u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91')
+        args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+                '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+                u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+                u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+                file)
+        headers = {'X-Header1': u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+                   'X-2': 1, 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:fg:lp'}
+
+        resp = MockHttpResponse()
+        conn[1].getresponse = resp.fake_response
+        conn[1].send = resp.fake_send
+        value = c.put_object(*args, headers=headers, http_conn=conn)
+        self.assertTrue(isinstance(value, basestring))
+        # Test for RFC-2616 encoded symbols
+        self.assertTrue("a-b: .x:yz mn:fg:lp" in resp.buffer[0],
+                      "[a-b: .x:yz mn:fg:lp] header is missing")
+
     def test_server_error(self):
         body = 'c' * 60
         c.http_connection = self.fake_http_connection(500, body=body)
@@ -378,6 +417,23 @@ class TestPostObject(MockHttpTest):
         args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', {})
         value = c.post_object(*args)
 
+    def test_unicode_ok(self):
+        conn = c.http_connection(u'http://www.test.com/')
+        args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+                '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+                u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+                u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91')
+        headers = {'X-Header1': u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+                   'X-2': 1, 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:kl:qr'}
+
+        resp = MockHttpResponse()
+        conn[1].getresponse = resp.fake_response
+        conn[1].send = resp.fake_send
+        c.post_object(*args, headers=headers, http_conn=conn)
+        # Test for RFC-2616 encoded symbols
+        self.assertTrue("a-b: .x:yz mn:kl:qr" in resp.buffer[0],
+                      "[a-b: .x:yz mn:kl:qr] header is missing")
+
     def test_server_error(self):
         body = 'c' * 60
         c.http_connection = self.fake_http_connection(500, body=body)