[Py34] Enable api.openstack.test_wsgi unit test

Issues:
- Method `webob.multidict.MultiDict.items` returns list object in py2 env[1]
  and listiterator in  py3 env[2]. ListIterator prevents changing values of
  headers in a loop(changes of item's order prevent skipping some headers).
- Statement `nova.utils.utf8(str(some_value))` did not work for bytes type in
  py3 env. Good example is `test_resource_headers_are_utf8` test. Int value 1
  became to b'b"b\'1\'"' after deserialization.
- Wrong checks for utf-8. (including check for encoding headers names, which
  is not encoded anywhere in the code)
- Wrong checks for `body` property of `webob.response.Response` cls. Method
  `_body__get` returns bytes[3]. There is no problem in python 2, since
  type(b"some") equals to type("some"), but it wrong for python 3.

[1] - https://github.com/Pylons/webob/blob/1.5.0b0/webob/multidict.py#L266-L267
[2] - https://github.com/Pylons/webob/blob/1.5.0b0/webob/multidict.py#L260-L264
[3] - https://github.com/Pylons/webob/blob/1.5.0b0/webob/response.py#L350

Related bp nova-python3-mitaka

Change-Id: I15150c8c8f204081b9b5d8bc28e622692d7d749a

nova/api/openstack/wsgi.py

@@ -297,7 +297,7 @@ class JSONDictSerializer(DictSerializer):
     """Default JSON request body serialization."""
 
     def default(self, data):
-        return jsonutils.dumps(data)
+        return six.text_type(jsonutils.dumps(data))
 
 
 def serializers(**serializers):
@@ -457,14 +457,20 @@ class ResponseObject(object):
                                                       default_serializers)
             serializer = _serializer()
 
-        response = webob.Response()
+        body = None
+        if self.obj is not None:
+            body = serializer.serialize(self.obj)
+        response = webob.Response(body=body)
+        if response.headers.get('Content-Length'):
+            # NOTE(andreykurilin): we need to encode 'Content-Length' header,
+            # since webob.Response auto sets it if "body" attr is presented.
+            # https://github.com/Pylons/webob/blob/1.5.0b0/webob/response.py#L147
+            response.headers['Content-Length'] = utils.utf8(
+                response.headers['Content-Length'])
         response.status_int = self.code
         for hdr, value in self._headers.items():
-            response.headers[hdr] = utils.utf8(str(value))
+            response.headers[hdr] = utils.utf8(value)
         response.headers['Content-Type'] = utils.utf8(content_type)
-        if self.obj is not None:
-            response.body = serializer.serialize(self.obj)
-
         return response
 
     @property
@@ -645,7 +651,7 @@ class Resource(wsgi.Application):
             content_type = request.get_content_type()
         except exception.InvalidContentType:
             LOG.debug("Unrecognized Content-Type provided in request")
-            return None, ''
+            return None, b''
 
         return content_type, request.body
 
@@ -860,10 +866,9 @@ class Resource(wsgi.Application):
                                               self.default_serializers)
 
         if hasattr(response, 'headers'):
-
+            for hdr, val in list(response.headers.items()):
-            for hdr, val in response.headers.items():
                 # Headers must be utf-8 strings
-                response.headers[hdr] = utils.utf8(str(val))
+                response.headers[hdr] = utils.utf8(val)
 
             if not request.api_version_request.is_null():
                 response.headers[API_VERSION_REQUEST_HEADER] = \
@@ -1158,7 +1163,7 @@ class Fault(webob.exc.HTTPException):
     def __init__(self, exception):
         """Create a Fault for the given webob.exc.exception."""
         self.wrapped_exc = exception
-        for key, value in self.wrapped_exc.headers.items():
+        for key, value in list(self.wrapped_exc.headers.items()):
             self.wrapped_exc.headers[key] = str(value)
         self.status_int = exception.status_int
 
@@ -1195,8 +1200,9 @@ class Fault(webob.exc.HTTPException):
             'application/json': JSONDictSerializer(),
         }[content_type]
 
-        self.wrapped_exc.body = serializer.serialize(fault_data)
         self.wrapped_exc.content_type = content_type
+        self.wrapped_exc.charset = 'UTF-8'
+        self.wrapped_exc.text = serializer.serialize(fault_data)
 
         return self.wrapped_exc
 
@@ -1245,7 +1251,8 @@ class RateLimitFault(webob.exc.HTTPException):
         }[content_type]
 
         content = serializer.serialize(self.content)
-        self.wrapped_exc.body = content
+        self.wrapped_exc.charset = 'UTF-8'
         self.wrapped_exc.content_type = content_type
+        self.wrapped_exc.text = content
 
         return self.wrapped_exc

nova/tests/unit/api/openstack/test_wsgi.py

@@ -23,6 +23,7 @@ from nova import exception
 from nova import i18n
 from nova import test
 from nova.tests.unit.api.openstack import fakes
+from nova.tests.unit import matchers
 from nova.tests.unit import utils
 from oslo_serialization import jsonutils
 
@@ -32,13 +33,13 @@ class RequestTest(test.NoDBTestCase):
 
     def test_content_type_missing(self):
         request = wsgi.Request.blank('/tests/123', method='POST')
-        request.body = "<body />"
+        request.body = b"<body />"
         self.assertIsNone(request.get_content_type())
 
     def test_content_type_unsupported(self):
         request = wsgi.Request.blank('/tests/123', method='POST')
         request.headers["Content-Type"] = "text/html"
-        request.body = "asdf<br />"
+        request.body = b"asdf<br />"
         self.assertRaises(exception.InvalidContentType,
                           request.get_content_type)
 
@@ -250,7 +251,7 @@ class JSONDeserializerTest(test.NoDBTestCase):
         self.assertEqual(deserializer.deserialize(data), as_dict)
 
     def test_json_valid_utf8(self):
-        data = """{"server": {"min_count": 1, "flavorRef": "1",
+        data = b"""{"server": {"min_count": 1, "flavorRef": "1",
                 "name": "\xe6\xa6\x82\xe5\xbf\xb5",
                 "imageRef": "10bab10c-1304-47d",
                 "max_count": 1}} """
@@ -269,7 +270,7 @@ class JSONDeserializerTest(test.NoDBTestCase):
 
     def test_json_invalid_utf8(self):
         """Send invalid utf-8 to JSONDeserializer."""
-        data = """{"server": {"min_count": 1, "flavorRef": "1",
+        data = b"""{"server": {"min_count": 1, "flavorRef": "1",
                 "name": "\xf0\x28\x8c\x28",
                 "imageRef": "10bab10c-1304-47d",
                 "max_count": 1}} """
@@ -301,7 +302,7 @@ class ResourceTest(test.NoDBTestCase):
         app = fakes.TestRouterV21(Controller())
         req = webob.Request.blank('/tests')
         response = req.get_response(app)
-        self.assertEqual(response.body, 'success')
+        self.assertEqual(b'success', response.body)
         self.assertEqual(response.status_int, 200)
 
     @mock.patch("nova.api.openstack.api_version_request.max_api_version")
@@ -320,7 +321,7 @@ class ResourceTest(test.NoDBTestCase):
         req = webob.Request.blank('/tests')
         req.headers = {self.header_name: version}
         response = req.get_response(app)
-        self.assertEqual(response.body, 'success')
+        self.assertEqual(b'success', response.body)
         self.assertEqual(response.status_int, 200)
 
     def test_resource_receives_api_version_request_invalid(self):
@@ -345,15 +346,15 @@ class ResourceTest(test.NoDBTestCase):
         # the default method is GET
         req = webob.Request.blank('/tests')
         response = req.get_response(app)
-        self.assertEqual(response.body, 'success')
+        self.assertEqual(b'success', response.body)
         self.assertEqual(response.status_int, 200)
-        req.body = '{"body": {"key": "value"}}'
+        req.body = b'{"body": {"key": "value"}}'
         response = req.get_response(app)
-        self.assertEqual(response.body, 'success')
+        self.assertEqual(b'success', response.body)
         self.assertEqual(response.status_int, 200)
         req.content_type = 'application/json'
         response = req.get_response(app)
-        self.assertEqual(response.body, 'success')
+        self.assertEqual(b'success', response.body)
         self.assertEqual(response.status_int, 200)
 
     def test_resource_call_with_method_post(self):
@@ -368,20 +369,20 @@ class ResourceTest(test.NoDBTestCase):
         app = fakes.TestRouter(Controller())
         req = webob.Request.blank('/tests', method="POST",
                                   content_type='application/json')
-        req.body = '{"body": {"key": "value"}}'
+        req.body = b'{"body": {"key": "value"}}'
         expected_body = {'body': {
             "key": "value"
             }
         }
         response = req.get_response(app)
         self.assertEqual(response.status_int, 200)
-        self.assertEqual(response.body, 'success')
+        self.assertEqual(b'success', response.body)
         # verify without body
         expected_body = None
         req.body = None
         response = req.get_response(app)
         self.assertEqual(response.status_int, 200)
-        self.assertEqual(response.body, 'success')
+        self.assertEqual(b'success', response.body)
         # the body is validated in the controller
         expected_body = {'body': None}
         response = req.get_response(app)
@@ -402,13 +403,13 @@ class ResourceTest(test.NoDBTestCase):
         app = fakes.TestRouter(Controller())
         req = webob.Request.blank('/tests/test_id', method="PUT",
                                   content_type='application/json')
-        req.body = '{"body": {"key": "value"}}'
+        req.body = b'{"body": {"key": "value"}}'
         expected_body = {'body': {
             "key": "value"
             }
         }
         response = req.get_response(app)
-        self.assertEqual(response.body, 'success')
+        self.assertEqual(b'success', response.body)
         self.assertEqual(response.status_int, 200)
         req.body = None
         expected_body = None
@@ -416,7 +417,7 @@ class ResourceTest(test.NoDBTestCase):
         self.assertEqual(response.status_int, 200)
         # verify no content_type is contained in the request
         req.content_type = None
-        req.body = '{"body": {"key": "value"}}'
+        req.body = b'{"body": {"key": "value"}}'
         response = req.get_response(app)
         expected_unsupported_type_body = {'badRequest':
             {'message': 'Unsupported Content-Type', 'code': 400}}
@@ -434,12 +435,12 @@ class ResourceTest(test.NoDBTestCase):
         req = webob.Request.blank('/tests/test_id', method="DELETE")
         response = req.get_response(app)
         self.assertEqual(response.status_int, 200)
-        self.assertEqual(response.body, 'success')
+        self.assertEqual(b'success', response.body)
         # ignore the body
-        req.body = '{"body": {"key": "value"}}'
+        req.body = b'{"body": {"key": "value"}}'
         response = req.get_response(app)
         self.assertEqual(response.status_int, 200)
-        self.assertEqual(response.body, 'success')
+        self.assertEqual(b'success', response.body)
 
     def test_resource_not_authorized(self):
         class Controller(object):
@@ -552,11 +553,11 @@ class ResourceTest(test.NoDBTestCase):
 
         request = wsgi.Request.blank('/', method='POST')
         request.headers['Content-Type'] = 'application/none'
-        request.body = 'foo'
+        request.body = b'foo'
 
         content_type, body = resource.get_body(request)
         self.assertIsNone(content_type)
-        self.assertEqual(body, '')
+        self.assertEqual(b'', body)
 
     def test_get_body_no_content_type(self):
         class Controller(object):
@@ -567,11 +568,11 @@ class ResourceTest(test.NoDBTestCase):
         resource = wsgi.Resource(controller)
 
         request = wsgi.Request.blank('/', method='POST')
-        request.body = 'foo'
+        request.body = b'foo'
 
         content_type, body = resource.get_body(request)
         self.assertIsNone(content_type)
-        self.assertEqual(body, 'foo')
+        self.assertEqual(b'foo', body)
 
     def test_get_body_no_content_body(self):
         class Controller(object):
@@ -583,11 +584,11 @@ class ResourceTest(test.NoDBTestCase):
 
         request = wsgi.Request.blank('/', method='POST')
         request.headers['Content-Type'] = 'application/json'
-        request.body = ''
+        request.body = b''
 
         content_type, body = resource.get_body(request)
         self.assertEqual('application/json', content_type)
-        self.assertEqual(body, '')
+        self.assertEqual(b'', body)
 
     def test_get_body(self):
         class Controller(object):
@@ -599,11 +600,11 @@ class ResourceTest(test.NoDBTestCase):
 
         request = wsgi.Request.blank('/', method='POST')
         request.headers['Content-Type'] = 'application/json'
-        request.body = 'foo'
+        request.body = b'foo'
 
         content_type, body = resource.get_body(request)
         self.assertEqual(content_type, 'application/json')
-        self.assertEqual(body, 'foo')
+        self.assertEqual(b'foo', body)
 
     def test_get_request_id_with_dict_response_body(self):
         class Controller(wsgi.Controller):
@@ -614,7 +615,7 @@ class ResourceTest(test.NoDBTestCase):
         app = fakes.TestRouter(Controller())
         response = req.get_response(app)
         self.assertIn('nova.context', req.environ)
-        self.assertEqual(response.body, '{"foo": "bar"}')
+        self.assertEqual(b'{"foo": "bar"}', response.body)
         self.assertEqual(response.status_int, 200)
 
     def test_no_request_id_with_str_response_body(self):
@@ -630,7 +631,7 @@ class ResourceTest(test.NoDBTestCase):
         # our wsgi setup, ideally it would be there.
         expected_header = self.get_req_id_header_name(req)
         self.assertFalse(hasattr(response.headers, expected_header))
-        self.assertEqual(response.body, 'foo')
+        self.assertEqual(b'foo', response.body)
         self.assertEqual(response.status_int, 200)
 
     def test_get_request_id_no_response_body(self):
@@ -642,7 +643,7 @@ class ResourceTest(test.NoDBTestCase):
         app = fakes.TestRouter(Controller())
         response = req.get_response(app)
         self.assertIn('nova.context', req.environ)
-        self.assertEqual(response.body, '')
+        self.assertEqual(b'', response.body)
         self.assertEqual(response.status_int, 200)
 
     def test_deserialize_badtype(self):
@@ -1006,14 +1007,12 @@ class ResourceTest(test.NoDBTestCase):
         req = webob.Request.blank('/tests')
         app = fakes.TestRouter(Controller())
         response = req.get_response(app)
-
+        for val in six.itervalues(response.headers):
-        for hdr, val in six.iteritems(response.headers):
             # All headers must be utf8
-            self.assertIsInstance(hdr, str)
+            self.assertThat(val, matchers.EncodedByUTF8())
-            self.assertIsInstance(val, str)
+        self.assertEqual(b'1', response.headers['x-header1'])
-        self.assertEqual(response.headers['x-header1'], '1')
+        self.assertEqual(b'header2', response.headers['x-header2'])
-        self.assertEqual(response.headers['x-header2'], 'header2')
+        self.assertEqual(b'header3', response.headers['x-header3'])
-        self.assertEqual(response.headers['x-header3'], 'header3')
 
     def test_resource_valid_utf8_body(self):
         class Controller(object):
@@ -1021,8 +1020,8 @@ class ResourceTest(test.NoDBTestCase):
                 return body
 
         req = webob.Request.blank('/tests/test_id', method="PUT")
-        body = """ {"name": "\xe6\xa6\x82\xe5\xbf\xb5" } """
+        body = b""" {"name": "\xe6\xa6\x82\xe5\xbf\xb5" } """
-        expected_body = '{"name": "\\u6982\\u5ff5"}'
+        expected_body = b'{"name": "\\u6982\\u5ff5"}'
         req.body = body
         req.headers['Content-Type'] = 'application/json'
         app = fakes.TestRouter(Controller())
@@ -1036,7 +1035,7 @@ class ResourceTest(test.NoDBTestCase):
                 return body
 
         req = webob.Request.blank('/tests/test_id', method="PUT")
-        body = """ {"name": "\xf0\x28\x8c\x28" } """
+        body = b""" {"name": "\xf0\x28\x8c\x28" } """
         req.body = body
         req.headers['Content-Type'] = 'application/json'
         app = fakes.TestRouter(Controller())
@@ -1130,17 +1129,16 @@ class ResponseObjectTest(test.NoDBTestCase):
         for content_type, mtype in wsgi._MEDIA_TYPE_MAP.items():
             request = wsgi.Request.blank('/tests/123')
             response = robj.serialize(request, content_type)
-
+            self.assertEqual(content_type.encode("utf-8"),
-            self.assertEqual(response.headers['Content-Type'], content_type)
+                             response.headers['Content-Type'])
             for hdr, val in six.iteritems(response.headers):
                 # All headers must be utf8
-                self.assertIsInstance(hdr, str)
+                self.assertThat(val, matchers.EncodedByUTF8())
-                self.assertIsInstance(val, str)
+            self.assertEqual(b'header1', response.headers['X-header1'])
-            self.assertEqual(response.headers['X-header1'], 'header1')
+            self.assertEqual(b'header2', response.headers['X-header2'])
-            self.assertEqual(response.headers['X-header2'], 'header2')
+            self.assertEqual(b'3', response.headers['X-header3'])
-            self.assertEqual(response.headers['X-header3'], '3')
             self.assertEqual(response.status_int, 202)
-            self.assertEqual(response.body, mtype)
+            self.assertEqual(mtype.encode("utf-8"), response.body)
 
 
 class ValidBodyTest(test.NoDBTestCase):

nova/tests/unit/matchers.py

@@ -21,6 +21,7 @@ import pprint
 from lxml import etree
 import six
 from testtools import content
+import testtools.matchers
 
 
 class DictKeysMismatch(object):
@@ -528,3 +529,23 @@ class XMLMatches(object):
                                               actual_child_idx)
         # The nodes match
         return True
+
+
+class EncodedByUTF8(object):
+    def match(self, obj):
+        if isinstance(obj, six.binary_type):
+            if hasattr(obj, "decode"):
+                try:
+                    obj.decode("utf-8")
+                except UnicodeDecodeError:
+                    return testtools.matchers.Mismatch(
+                        "%s is not encoded in UTF-8." % obj)
+        else:
+            reason = ("Type of '%(obj)s' is '%(obj_type)s', "
+                      "should be '%(correct_type)s'."
+                      % {
+                          "obj": obj,
+                          "obj_type": type(obj).__name__,
+                          "correct_type": six.binary_type.__name__
+                      })
+            return testtools.matchers.Mismatch(reason)

nova/tests/unit/test_utils.py

@@ -1374,3 +1374,24 @@ class SpawnTestCase(SpawnNTestCase):
     def setUp(self):
         super(SpawnTestCase, self).setUp()
         self.spawn_name = 'spawn'
+
+
+class UT8TestCase(test.NoDBTestCase):
+    def test_none_value(self):
+        self.assertIsInstance(utils.utf8(None), type(None))
+
+    def test_bytes_value(self):
+        some_value = b"fake data"
+        return_value = utils.utf8(some_value)
+        # check that type of returned value doesn't changed
+        self.assertIsInstance(return_value, type(some_value))
+        self.assertEqual(some_value, return_value)
+
+    def test_not_text_type(self):
+        return_value = utils.utf8(1)
+        self.assertEqual(b"1", return_value)
+        self.assertIsInstance(return_value, six.binary_type)
+
+    def test_text_type_with_encoding(self):
+        some_value = 'test\u2026config'
+        self.assertEqual(some_value, utils.utf8(some_value).decode("utf-8"))

nova/utils.py

@@ -587,14 +587,17 @@ def xhtml_escape(value):
 def utf8(value):
     """Try to turn a string into utf-8 if possible.
 
-    Code is directly from the utf8 function in
+    The original code was copied from the utf8 function in
     http://github.com/facebook/tornado/blob/master/tornado/escape.py
 
     """
-    if isinstance(value, six.text_type):
+    if value is None or isinstance(value, six.binary_type):
-        return value.encode('utf-8')
+        return value
-    assert isinstance(value, str)
+
-    return value
+    if not isinstance(value, six.text_type):
+        value = six.text_type(value)
+
+    return value.encode('utf-8')
 
 
 def check_isinstance(obj, cls):

tests-py3.txt

@@ -140,11 +140,6 @@ nova.tests.unit.api.openstack.compute.test_volumes.VolumeApiTestV21
 nova.tests.unit.api.openstack.compute.test_volumes.VolumeAttachTestsV2
 nova.tests.unit.api.openstack.compute.test_volumes.VolumeAttachTestsV21
 nova.tests.unit.api.openstack.test_faults.TestFaultWrapper
-nova.tests.unit.api.openstack.test_faults.TestFaults
-nova.tests.unit.api.openstack.test_wsgi.JSONDeserializerTest
-nova.tests.unit.api.openstack.test_wsgi.RequestTest
-nova.tests.unit.api.openstack.test_wsgi.ResourceTest
-nova.tests.unit.api.openstack.test_wsgi.ResponseObjectTest
 nova.tests.unit.api.test_compute_req_id.RequestIdTest
 nova.tests.unit.api.test_validator.ValidatorTestCase
 nova.tests.unit.api.test_wsgi.Test