From 8b42f8a40c9a48f85b8d4d859afb4e28a510c036 Mon Sep 17 00:00:00 2001
From: Tihomir Trifonov <t.trifonov@gmail.com>
Date: Thu, 11 Oct 2012 15:04:00 +0300
Subject: [PATCH] Force utf-8 encode of HTTPConnection params
This patch forces swiftclient to encode to utf-8
all url and headers arguments, to avoid the
UnicodeDecodeError which is raised by '\r\n'.join([])
invoked in htplib.py.
Currently the affected projects are Horizon(upload file
with unicode name) and swiftclient CLI('swift post' with
unicode filename as header)
This is also a follow-up of this review:
https://review.openstack.org/#/c/14216/
I'd still want to hear what the Swift core devs
think of it. Is it better to create a new
AutoEncodingHTTPConnection? Or to handle the connection
creation and make sure there are no unicode and utf-8
string at the same time. If these unicode checks have to
be added in the calling code(Dashboard, CLI), there are
so many places to be added, and also in all new commands
that might be exposed from the API.
Fixes bug 1008940
Change-Id: Ice2aa29024429d3e6f569a88d5cf8b4202537827
---
swiftclient/client.py | 31 +++++++++++++++++++++-
tests/test_swiftclient.py | 56 +++++++++++++++++++++++++++++++++++++++
2 files changed, 86 insertions(+), 1 deletion(-)
diff --git a/swiftclient/client.py b/swiftclient/client.py
index 896dd354..ff35f652 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -20,6 +20,7 @@ Cloud Files client library used internally
import socket
import os
import logging
+from functools import wraps
from urllib import quote as _quote
from urlparse import urlparse, urlunparse
@@ -81,9 +82,17 @@ def quote(value, safe='/'):
"""
Patched version of urllib.quote that encodes utf8 strings before quoting
"""
+ value = encode_utf8(value)
+ if isinstance(value, str):
+ return _quote(value, safe)
+ else:
+ return value
+
+
+def encode_utf8(value):
if isinstance(value, unicode):
value = value.encode('utf8')
- return _quote(value, safe)
+ return value
# look for a real json parser first
@@ -161,6 +170,7 @@ def http_connection(url, proxy=None):
:returns: tuple of (parsed url, connection object)
:raises ClientException: Unable to handle protocol scheme
"""
+ url = encode_utf8(url)
parsed = urlparse(url)
proxy_parsed = urlparse(proxy) if proxy else None
if parsed.scheme == 'http':
@@ -170,6 +180,25 @@ def http_connection(url, proxy=None):
else:
raise ClientException('Cannot handle protocol scheme %s for url %s' %
(parsed.scheme, repr(url)))
+
+ def putheader_wrapper(func):
+
+ @wraps(func)
+ def putheader_escaped(key, value):
+ func(encode_utf8(key), encode_utf8(value))
+ return putheader_escaped
+ conn.putheader = putheader_wrapper(conn.putheader)
+
+ def request_wrapper(func):
+
+ @wraps(func)
+ def request_escaped(method, url, body=None, headers=None):
+ url = encode_utf8(url)
+ if body:
+ body = encode_utf8(body)
+ func(method, url, body=body, headers=headers or {})
+ return request_escaped
+ conn.request = request_wrapper(conn.request)
if proxy:
conn._set_tunnel(parsed.hostname, parsed.port)
return parsed, conn
diff --git a/tests/test_swiftclient.py b/tests/test_swiftclient.py
index 121456c1..aad6e249 100644
--- a/tests/test_swiftclient.py
+++ b/tests/test_swiftclient.py
@@ -15,6 +15,7 @@
# TODO: More tests
import socket
+import StringIO
import unittest
from urlparse import urlparse
@@ -121,6 +122,24 @@ class MockHttpTest(unittest.TestCase):
reload(c)
+class MockHttpResponse():
+ def __init__(self):
+ self.status = 200
+ self.buffer = []
+
+ def read(self):
+ return ""
+
+ def getheader(self, name, default):
+ return ""
+
+ def fake_response(self):
+ return MockHttpResponse()
+
+ def fake_send(self, msg):
+ self.buffer.append(msg)
+
+
class TestHttpHelpers(MockHttpTest):
def test_quote(self):
@@ -360,6 +379,26 @@ class TestPutObject(MockHttpTest):
value = c.put_object(*args)
self.assertTrue(isinstance(value, basestring))
+ def test_unicode_ok(self):
+ conn = c.http_connection(u'http://www.test.com/')
+ file = StringIO.StringIO(u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91')
+ args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+ '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+ u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+ u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+ file)
+ headers = {'X-Header1': u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+ 'X-2': 1, 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:fg:lp'}
+
+ resp = MockHttpResponse()
+ conn[1].getresponse = resp.fake_response
+ conn[1].send = resp.fake_send
+ value = c.put_object(*args, headers=headers, http_conn=conn)
+ self.assertTrue(isinstance(value, basestring))
+ # Test for RFC-2616 encoded symbols
+ self.assertTrue("a-b: .x:yz mn:fg:lp" in resp.buffer[0],
+ "[a-b: .x:yz mn:fg:lp] header is missing")
+
def test_server_error(self):
body = 'c' * 60
c.http_connection = self.fake_http_connection(500, body=body)
@@ -378,6 +417,23 @@ class TestPostObject(MockHttpTest):
args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', {})
value = c.post_object(*args)
+ def test_unicode_ok(self):
+ conn = c.http_connection(u'http://www.test.com/')
+ args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+ '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+ u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+ u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91')
+ headers = {'X-Header1': u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
+ 'X-2': 1, 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:kl:qr'}
+
+ resp = MockHttpResponse()
+ conn[1].getresponse = resp.fake_response
+ conn[1].send = resp.fake_send
+ c.post_object(*args, headers=headers, http_conn=conn)
+ # Test for RFC-2616 encoded symbols
+ self.assertTrue("a-b: .x:yz mn:kl:qr" in resp.buffer[0],
+ "[a-b: .x:yz mn:kl:qr] header is missing")
+
def test_server_error(self):
body = 'c' * 60
c.http_connection = self.fake_http_connection(500, body=body)