Properly decode body before calling json.loads()

This commit fixes an issue when running the rest client and token
clients on python 3. httplib2 sets the type for the response body as
'bytes' when running on python 3, which requires that it be decoded
prior to running json.loads() on it. Additionally, in the v2 token
client a type check was done on the response body which was no longer
evaluating true because the type was no longer a 'str'. This was
fixed as part of the broader cleanup.

Change-Id: If4d496c4f10cec7d7050afc4b07f1f263de4c3e5

tempest_lib/common/rest_client.py

@@ -439,9 +439,16 @@ class RestClient(object):
             self._log_request_full(method, req_url, resp, secs, req_headers,
                                    req_body, resp_body, caller_name, extra)
 
+    def _json_loads(self, resp_body):
+        if isinstance(resp_body, bytes):
+            resp_body = json.loads(resp_body.decode('utf8'))
+        else:
+            resp_body = json.loads(resp_body)
+        return resp_body
+
     def _parse_resp(self, body):
         try:
-            body = json.loads(body)
+            body = self._json_loads(body)
         except ValueError:
             return body
 

tempest_lib/services/identity/v2/token_client.py

@@ -85,15 +85,13 @@ class TokenClientJSON(rest_client.RestClient):
         self._log_request(method, url, resp)
 
         if resp.status in [401, 403]:
-            resp_body = json.loads(resp_body)
+            resp_body = self._json_loads(resp_body)
             raise exceptions.Unauthorized(resp_body['error']['message'])
         elif resp.status not in [200, 201]:
             raise exceptions.IdentityError(
                 'Unexpected status code {0}'.format(resp.status))
 
-        if isinstance(resp_body, str):
+        return resp, self._json_loads(resp_body)
-            resp_body = json.loads(resp_body)
-        return resp, resp_body
 
     def get_token(self, user, password, tenant, auth_data=False):
         """Returns (token id, token data) for supplied credentials."""

tempest_lib/services/identity/v3/token_client.py

@@ -135,13 +135,13 @@ class V3TokenClientJSON(rest_client.RestClient):
         self._log_request(method, url, resp)
 
         if resp.status in [401, 403]:
-            resp_body = json.loads(resp_body)
+            resp_body = self._json_loads(resp_body)
             raise exceptions.Unauthorized(resp_body['error']['message'])
         elif resp.status not in [200, 201, 204]:
             raise exceptions.IdentityError(
                 'Unexpected status code {0}'.format(resp.status))
 
-        return resp, json.loads(resp_body)
+        return resp, self._json_loads(resp_body)
 
     def get_token(self, **kwargs):
         """Returns (token id, token data) for supplied credentials"""

tempest_lib/tests/services/identity/v2/test_token_client.py

@@ -14,6 +14,7 @@
 
 import json
 
+import httplib2
 from oslotest import mockpatch
 
 from tempest_lib.common import rest_client
@@ -64,3 +65,23 @@ class TestTokenClientV2(base.TestCase):
         })
         post_mock.mock.assert_called_once_with('fake_url/tokens',
                                                body=req_dict)
+
+    def test_request_with_str_body(self):
+        token_client_v2 = token_client.TokenClientJSON('fake_url')
+        self.useFixture(mockpatch.PatchObject(
+            token_client_v2, 'raw_request', return_value=(
+                httplib2.Response({'status': '200'}),
+                str('{"access": {"token": "fake_token"}}'))))
+        resp, body = token_client_v2.request('GET', 'fake_uri')
+        self.assertIsInstance(resp, httplib2.Response)
+        self.assertIsInstance(body, dict)
+
+    def test_request_with_bytes_body(self):
+        token_client_v2 = token_client.TokenClientJSON('fake_url')
+        self.useFixture(mockpatch.PatchObject(
+            token_client_v2, 'raw_request', return_value=(
+                httplib2.Response({'status': '200'}),
+                bytes(b'{"access": {"token": "fake_token"}}'))))
+        resp, body = token_client_v2.request('GET', 'fake_uri')
+        self.assertIsInstance(resp, httplib2.Response)
+        self.assertIsInstance(body, dict)

tempest_lib/tests/services/identity/v3/test_token_client.py

@@ -14,6 +14,7 @@
 
 import json
 
+import httplib2
 from oslotest import mockpatch
 
 from tempest_lib.common import rest_client
@@ -79,3 +80,23 @@ class TestTokenClientV2(base.TestCase):
 
         post_mock.mock.assert_called_once_with('fake_url/auth/tokens',
                                                body=req_dict)
+
+    def test_request_with_str_body(self):
+        token_client_v3 = token_client.V3TokenClientJSON('fake_url')
+        self.useFixture(mockpatch.PatchObject(
+            token_client_v3, 'raw_request', return_value=(
+                httplib2.Response({"status": "200"}),
+                str('{"access": {"token": "fake_token"}}'))))
+        resp, body = token_client_v3.request('GET', 'fake_uri')
+        self.assertIsInstance(resp, httplib2.Response)
+        self.assertIsInstance(body, dict)
+
+    def test_request_with_bytes_body(self):
+        token_client_v3 = token_client.V3TokenClientJSON('fake_url')
+        self.useFixture(mockpatch.PatchObject(
+            token_client_v3, 'raw_request', return_value=(
+                httplib2.Response({"status": "200"}),
+                bytes(b'{"access": {"token": "fake_token"}}'))))
+        resp, body = token_client_v3.request('GET', 'fake_uri')
+        self.assertIsInstance(resp, httplib2.Response)
+        self.assertIsInstance(body, dict)