Accept gzip-encoded API responses

Previously, we would accept gzip-encoded responses, but only because we
were letting requests decode *all* responses (even object data). This
restores the previous capability, but with tighter controls about which
requests will accept gzipped responses and where the decoding happens.

Change-Id: I4fd8b97207b9ab01b1bcf825cc16efd8ad46344a
Related-Bug: 1282861
Related-Bug: 1338464

swiftclient/client.py

@@ -732,7 +732,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
     if end_marker:
         qs += '&end_marker=%s' % quote(end_marker)
     full_path = '%s?%s' % (parsed.path, qs)
-    headers = {'X-Auth-Token': token}
+    headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'}
     if service_token:
         headers['X-Service-Token'] = service_token
     method = 'GET'
@@ -859,6 +859,7 @@ def get_container(url, token, container, marker=None, limit=None,
     else:
         headers = {}
     headers['X-Auth-Token'] = token
+    headers['Accept-Encoding'] = 'gzip'
     if full_listing:
         rv = get_container(url, token, container, marker, limit, prefix,
                            delimiter, end_marker, path, http_conn,
@@ -1457,10 +1458,11 @@ def get_capabilities(http_conn):
     :raises ClientException: HTTP Capabilities GET failed
     """
     parsed, conn = http_conn
-    conn.request('GET', parsed.path, '')
+    headers = {'Accept-Encoding': 'gzip'}
+    conn.request('GET', parsed.path, '', headers)
     resp = conn.getresponse()
     body = resp.read()
-    http_log((parsed.geturl(), 'GET',), {'headers': {}}, resp, body)
+    http_log((parsed.geturl(), 'GET',), {'headers': headers}, resp, body)
     if resp.status < 200 or resp.status >= 300:
         raise ClientException.from_response(
             resp, 'Capabilities GET failed', body)

swiftclient/utils.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Miscellaneous utility functions for use with Swift."""
+import gzip
 import hashlib
 import hmac
 import json
@@ -120,6 +121,10 @@ def generate_temp_url(path, seconds, key, method, absolute=False):
 
 
 def parse_api_response(headers, body):
+    if headers.get('content-encoding') == 'gzip':
+        with gzip.GzipFile(fileobj=six.BytesIO(body), mode='r') as gz:
+            body = gz.read()
+
     charset = 'utf-8'
     # Swift *should* be speaking UTF-8, but check content-type just in case
     content_type = headers.get('content-type', '')

tests/unit/test_swiftclient.py

@@ -581,6 +581,7 @@ class TestGetAccount(MockHttpTest):
         self.assertEqual(value, [])
         self.assertRequests([
             ('GET', '/v1/acct?format=json', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'asdf'}),
         ])
 
@@ -591,6 +592,7 @@ class TestGetAccount(MockHttpTest):
         c.get_account('http://www.test.com/v1/acct', 'asdf', marker='marker')
         self.assertRequests([
             ('GET', '/v1/acct?format=json&marker=marker', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'asdf'}),
         ])
 
@@ -601,6 +603,7 @@ class TestGetAccount(MockHttpTest):
         c.get_account('http://www.test.com/v1/acct', 'asdf', limit=10)
         self.assertRequests([
             ('GET', '/v1/acct?format=json&limit=10', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'asdf'}),
         ])
 
@@ -611,6 +614,7 @@ class TestGetAccount(MockHttpTest):
         c.get_account('http://www.test.com/v1/acct', 'asdf', prefix='asdf/')
         self.assertRequests([
             ('GET', '/v1/acct?format=json&prefix=asdf/', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'asdf'}),
         ])
 
@@ -622,6 +626,7 @@ class TestGetAccount(MockHttpTest):
                       end_marker='end_marker')
         self.assertRequests([
             ('GET', '/v1/acct?format=json&end_marker=end_marker', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'asdf'}),
         ])
 
@@ -700,6 +705,7 @@ class TestGetContainer(MockHttpTest):
         self.assertEqual(value, [])
         self.assertRequests([
             ('GET', '/v1/acct/container?format=json', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'token'}),
         ])
 
@@ -711,6 +717,7 @@ class TestGetContainer(MockHttpTest):
                         marker='marker')
         self.assertRequests([
             ('GET', '/v1/acct/container?format=json&marker=marker', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'token'}),
         ])
 
@@ -722,6 +729,7 @@ class TestGetContainer(MockHttpTest):
                         limit=10)
         self.assertRequests([
             ('GET', '/v1/acct/container?format=json&limit=10', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'token'}),
         ])
 
@@ -733,6 +741,7 @@ class TestGetContainer(MockHttpTest):
                         prefix='asdf/')
         self.assertRequests([
             ('GET', '/v1/acct/container?format=json&prefix=asdf/', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'token'}),
         ])
 
@@ -744,6 +753,7 @@ class TestGetContainer(MockHttpTest):
                         delimiter='/')
         self.assertRequests([
             ('GET', '/v1/acct/container?format=json&delimiter=/', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'token'}),
         ])
 
@@ -755,7 +765,7 @@ class TestGetContainer(MockHttpTest):
                         end_marker='end_marker')
         self.assertRequests([
             ('GET', '/v1/acct/container?format=json&end_marker=end_marker',
-             '', {'x-auth-token': 'token'}),
+             '', {'x-auth-token': 'token', 'accept-encoding': 'gzip'}),
         ])
 
     def test_param_path(self):
@@ -766,6 +776,7 @@ class TestGetContainer(MockHttpTest):
                         path='asdf')
         self.assertRequests([
             ('GET', '/v1/acct/container?format=json&path=asdf', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'token'}),
         ])
 
@@ -780,6 +791,7 @@ class TestGetContainer(MockHttpTest):
             ('GET', '/container?format=json', '', {
                 'x-auth-token': 'TOKEN',
                 'x-client-key': 'client key',
+                'accept-encoding': 'gzip',
             }),
         ])
 
@@ -790,6 +802,7 @@ class TestGetContainer(MockHttpTest):
                         query_string="hello=20")
         self.assertRequests([
             ('GET', '/asdf?format=json&hello=20', '', {
+                'accept-encoding': 'gzip',
                 'x-auth-token': 'asdf'}),
         ])
 
@@ -1583,7 +1596,7 @@ class TestGetCapabilities(MockHttpTest):
         http_conn = conn('http://www.test.com/info')
         info = c.get_capabilities(http_conn)
         self.assertRequests([
-            ('GET', '/info', '', {}),
+            ('GET', '/info', '', {'Accept-Encoding': 'gzip'}),
         ])
         self.assertEqual(info, {})
         self.assertTrue(http_conn[1].resp.has_been_read)
@@ -1619,7 +1632,8 @@ class TestGetCapabilities(MockHttpTest):
             ('GET', '/auth/v1.0', '', {
                 'x-auth-user': 'user',
                 'x-auth-key': 'key'}),
-            ('GET', 'http://storage.example.com/info', '', {}),
+            ('GET', 'http://storage.example.com/info', '', {
+                'accept-encoding': 'gzip'}),
         ])
 
     def test_conn_get_capabilities_with_os_auth(self):
@@ -2341,6 +2355,7 @@ class TestConnection(MockHttpTest):
             ('GET', '/v1/a/c1?format=json&limit=5&prefix=p', '', {
                 'x-auth-token': 'token',
                 'X-Favourite-Pet': 'Aardvark',
+                'accept-encoding': 'gzip',
             }),
         ])
         self.assertEqual(conn.attempts, 1)

tests/unit/test_utils.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import gzip
 import unittest
 import mock
 import six
@@ -394,3 +395,39 @@ class TestGroupers(unittest.TestCase):
 
         result = list(u.n_groups(range(100), 12))
         self.assertEqual([9] * 11 + [1], list(map(len, result)))
+
+
+class TestApiResponeParser(unittest.TestCase):
+
+    def test_utf8_default(self):
+        result = u.parse_api_response(
+            {}, u'{"test": "\u2603"}'.encode('utf8'))
+        self.assertEqual({'test': u'\u2603'}, result)
+
+        result = u.parse_api_response(
+            {}, u'{"test": "\\u2603"}'.encode('utf8'))
+        self.assertEqual({'test': u'\u2603'}, result)
+
+    def test_bad_json(self):
+        self.assertRaises(ValueError, u.parse_api_response,
+                          {}, b'{"foo": "bar}')
+
+    def test_bad_utf8(self):
+        self.assertRaises(UnicodeDecodeError, u.parse_api_response,
+                          {}, b'{"foo": "b\xffr"}')
+
+    def test_latin_1(self):
+        result = u.parse_api_response(
+            {'content-type': 'application/json; charset=iso8859-1'},
+            b'{"t\xe9st": "\xff"}')
+        self.assertEqual({u't\xe9st': u'\xff'}, result)
+
+    def test_gzipped_utf8(self):
+        buf = six.BytesIO()
+        gz = gzip.GzipFile(fileobj=buf, mode='w')
+        gz.write(u'{"test": "\u2603"}'.encode('utf8'))
+        gz.close()
+        result = u.parse_api_response(
+            {'content-encoding': 'gzip'},
+            buf.getvalue())
+        self.assertEqual({'test': u'\u2603'}, result)