Merge "Accept gzip-encoded API responses"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
92544c58c5
@ -739,7 +739,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
|
||||
if end_marker:
|
||||
qs += '&end_marker=%s' % quote(end_marker)
|
||||
full_path = '%s?%s' % (parsed.path, qs)
|
||||
headers = {'X-Auth-Token': token}
|
||||
headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'}
|
||||
if service_token:
|
||||
headers['X-Service-Token'] = service_token
|
||||
method = 'GET'
|
||||
@ -866,6 +866,7 @@ def get_container(url, token, container, marker=None, limit=None,
|
||||
else:
|
||||
headers = {}
|
||||
headers['X-Auth-Token'] = token
|
||||
headers['Accept-Encoding'] = 'gzip'
|
||||
if full_listing:
|
||||
rv = get_container(url, token, container, marker, limit, prefix,
|
||||
delimiter, end_marker, path, http_conn,
|
||||
@ -1464,10 +1465,11 @@ def get_capabilities(http_conn):
|
||||
:raises ClientException: HTTP Capabilities GET failed
|
||||
"""
|
||||
parsed, conn = http_conn
|
||||
conn.request('GET', parsed.path, '')
|
||||
headers = {'Accept-Encoding': 'gzip'}
|
||||
conn.request('GET', parsed.path, '', headers)
|
||||
resp = conn.getresponse()
|
||||
body = resp.read()
|
||||
http_log((parsed.geturl(), 'GET',), {'headers': {}}, resp, body)
|
||||
http_log((parsed.geturl(), 'GET',), {'headers': headers}, resp, body)
|
||||
if resp.status < 200 or resp.status >= 300:
|
||||
raise ClientException.from_response(
|
||||
resp, 'Capabilities GET failed', body)
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Miscellaneous utility functions for use with Swift."""
|
||||
import gzip
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
@ -120,6 +121,10 @@ def generate_temp_url(path, seconds, key, method, absolute=False):
|
||||
|
||||
|
||||
def parse_api_response(headers, body):
|
||||
if headers.get('content-encoding') == 'gzip':
|
||||
with gzip.GzipFile(fileobj=six.BytesIO(body), mode='r') as gz:
|
||||
body = gz.read()
|
||||
|
||||
charset = 'utf-8'
|
||||
# Swift *should* be speaking UTF-8, but check content-type just in case
|
||||
content_type = headers.get('content-type', '')
|
||||
|
||||
@ -582,6 +582,7 @@ class TestGetAccount(MockHttpTest):
|
||||
self.assertEqual(value, [])
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct?format=json', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'asdf'}),
|
||||
])
|
||||
|
||||
@ -592,6 +593,7 @@ class TestGetAccount(MockHttpTest):
|
||||
c.get_account('http://www.test.com/v1/acct', 'asdf', marker='marker')
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct?format=json&marker=marker', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'asdf'}),
|
||||
])
|
||||
|
||||
@ -602,6 +604,7 @@ class TestGetAccount(MockHttpTest):
|
||||
c.get_account('http://www.test.com/v1/acct', 'asdf', limit=10)
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct?format=json&limit=10', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'asdf'}),
|
||||
])
|
||||
|
||||
@ -612,6 +615,7 @@ class TestGetAccount(MockHttpTest):
|
||||
c.get_account('http://www.test.com/v1/acct', 'asdf', prefix='asdf/')
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct?format=json&prefix=asdf/', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'asdf'}),
|
||||
])
|
||||
|
||||
@ -623,6 +627,7 @@ class TestGetAccount(MockHttpTest):
|
||||
end_marker='end_marker')
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct?format=json&end_marker=end_marker', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'asdf'}),
|
||||
])
|
||||
|
||||
@ -701,6 +706,7 @@ class TestGetContainer(MockHttpTest):
|
||||
self.assertEqual(value, [])
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct/container?format=json', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'token'}),
|
||||
])
|
||||
|
||||
@ -712,6 +718,7 @@ class TestGetContainer(MockHttpTest):
|
||||
marker='marker')
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct/container?format=json&marker=marker', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'token'}),
|
||||
])
|
||||
|
||||
@ -723,6 +730,7 @@ class TestGetContainer(MockHttpTest):
|
||||
limit=10)
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct/container?format=json&limit=10', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'token'}),
|
||||
])
|
||||
|
||||
@ -734,6 +742,7 @@ class TestGetContainer(MockHttpTest):
|
||||
prefix='asdf/')
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct/container?format=json&prefix=asdf/', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'token'}),
|
||||
])
|
||||
|
||||
@ -745,6 +754,7 @@ class TestGetContainer(MockHttpTest):
|
||||
delimiter='/')
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct/container?format=json&delimiter=/', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'token'}),
|
||||
])
|
||||
|
||||
@ -756,7 +766,7 @@ class TestGetContainer(MockHttpTest):
|
||||
end_marker='end_marker')
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct/container?format=json&end_marker=end_marker',
|
||||
'', {'x-auth-token': 'token'}),
|
||||
'', {'x-auth-token': 'token', 'accept-encoding': 'gzip'}),
|
||||
])
|
||||
|
||||
def test_param_path(self):
|
||||
@ -767,6 +777,7 @@ class TestGetContainer(MockHttpTest):
|
||||
path='asdf')
|
||||
self.assertRequests([
|
||||
('GET', '/v1/acct/container?format=json&path=asdf', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'token'}),
|
||||
])
|
||||
|
||||
@ -781,6 +792,7 @@ class TestGetContainer(MockHttpTest):
|
||||
('GET', '/container?format=json', '', {
|
||||
'x-auth-token': 'TOKEN',
|
||||
'x-client-key': 'client key',
|
||||
'accept-encoding': 'gzip',
|
||||
}),
|
||||
])
|
||||
|
||||
@ -791,6 +803,7 @@ class TestGetContainer(MockHttpTest):
|
||||
query_string="hello=20")
|
||||
self.assertRequests([
|
||||
('GET', '/asdf?format=json&hello=20', '', {
|
||||
'accept-encoding': 'gzip',
|
||||
'x-auth-token': 'asdf'}),
|
||||
])
|
||||
|
||||
@ -1584,7 +1597,7 @@ class TestGetCapabilities(MockHttpTest):
|
||||
http_conn = conn('http://www.test.com/info')
|
||||
info = c.get_capabilities(http_conn)
|
||||
self.assertRequests([
|
||||
('GET', '/info', '', {}),
|
||||
('GET', '/info', '', {'Accept-Encoding': 'gzip'}),
|
||||
])
|
||||
self.assertEqual(info, {})
|
||||
self.assertTrue(http_conn[1].resp.has_been_read)
|
||||
@ -1620,7 +1633,8 @@ class TestGetCapabilities(MockHttpTest):
|
||||
('GET', '/auth/v1.0', '', {
|
||||
'x-auth-user': 'user',
|
||||
'x-auth-key': 'key'}),
|
||||
('GET', 'http://storage.example.com/info', '', {}),
|
||||
('GET', 'http://storage.example.com/info', '', {
|
||||
'accept-encoding': 'gzip'}),
|
||||
])
|
||||
|
||||
def test_conn_get_capabilities_with_os_auth(self):
|
||||
@ -2342,6 +2356,7 @@ class TestConnection(MockHttpTest):
|
||||
('GET', '/v1/a/c1?format=json&limit=5&prefix=p', '', {
|
||||
'x-auth-token': 'token',
|
||||
'X-Favourite-Pet': 'Aardvark',
|
||||
'accept-encoding': 'gzip',
|
||||
}),
|
||||
])
|
||||
self.assertEqual(conn.attempts, 1)
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import gzip
|
||||
import unittest
|
||||
import mock
|
||||
import six
|
||||
@ -394,3 +395,39 @@ class TestGroupers(unittest.TestCase):
|
||||
|
||||
result = list(u.n_groups(range(100), 12))
|
||||
self.assertEqual([9] * 11 + [1], list(map(len, result)))
|
||||
|
||||
|
||||
class TestApiResponeParser(unittest.TestCase):
|
||||
|
||||
def test_utf8_default(self):
|
||||
result = u.parse_api_response(
|
||||
{}, u'{"test": "\u2603"}'.encode('utf8'))
|
||||
self.assertEqual({'test': u'\u2603'}, result)
|
||||
|
||||
result = u.parse_api_response(
|
||||
{}, u'{"test": "\\u2603"}'.encode('utf8'))
|
||||
self.assertEqual({'test': u'\u2603'}, result)
|
||||
|
||||
def test_bad_json(self):
|
||||
self.assertRaises(ValueError, u.parse_api_response,
|
||||
{}, b'{"foo": "bar}')
|
||||
|
||||
def test_bad_utf8(self):
|
||||
self.assertRaises(UnicodeDecodeError, u.parse_api_response,
|
||||
{}, b'{"foo": "b\xffr"}')
|
||||
|
||||
def test_latin_1(self):
|
||||
result = u.parse_api_response(
|
||||
{'content-type': 'application/json; charset=iso8859-1'},
|
||||
b'{"t\xe9st": "\xff"}')
|
||||
self.assertEqual({u't\xe9st': u'\xff'}, result)
|
||||
|
||||
def test_gzipped_utf8(self):
|
||||
buf = six.BytesIO()
|
||||
gz = gzip.GzipFile(fileobj=buf, mode='w')
|
||||
gz.write(u'{"test": "\u2603"}'.encode('utf8'))
|
||||
gz.close()
|
||||
result = u.parse_api_response(
|
||||
{'content-encoding': 'gzip'},
|
||||
buf.getvalue())
|
||||
self.assertEqual({'test': u'\u2603'}, result)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user