[PY3] byte/string conversions and enable PY3 test

* The dict.items()[0] will raise a TypeError in PY3,
  as dict.items() doesn't return a list any more in PY3
  but a view of list.
* Webob response body should be bytes not strings so used
  oslo_utils.encodeutils.safe_decode to decode it.

Partially implements blueprint: goal-python35
Co-Authored-By: ChangBo Guo(gcb) <eric.guo@easystack.cn>

Change-Id: I38d416923bc0cec0ca98c4494dd1e06cd49671cf

nova/api/openstack/compute/volumes.py

@@ -70,7 +70,7 @@ def _translate_volume_summary_view(context, vol):
         #                   'mountpoint': '/dev/sda/
         #                    }
         #                }
-        attachment = vol['attachments'].items()[0]
+        attachment = list(vol['attachments'].items())[0]
         d['attachments'] = [_translate_attachment_detail_view(vol['id'],
             attachment[0],
             attachment[1].get('mountpoint'))]

nova/api/openstack/wsgi.py

@@ -19,6 +19,7 @@ import functools
 import microversion_parse
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
+from oslo_utils import encodeutils
 from oslo_utils import strutils
 import six
 import webob
@@ -692,7 +693,8 @@ class Resource(wsgi.Application):
         if hasattr(response, 'headers'):
             for hdr, val in list(response.headers.items()):
                 # Headers must be utf-8 strings
-                response.headers[hdr] = utils.utf8(val)
+                response.headers[hdr] = encodeutils.safe_decode(
+                        utils.utf8(val))
 
             if not request.api_version_request.is_null():
                 response.headers[API_VERSION_REQUEST_HEADER] = \

nova/tests/unit/api/openstack/compute/test_api.py

@@ -14,6 +14,7 @@
 #    under the License.
 
 from oslo_serialization import jsonutils
+from oslo_utils import encodeutils
 import six
 import webob.dec
 import webob.exc
@@ -69,7 +70,8 @@ class APITest(test.NoDBTestCase):
         # Consider changing `api.openstack.wsgi.Resource._process_stack`
         # to encode header values in ASCII rather than UTF-8.
         # https://tools.ietf.org/html/rfc7230#section-3.2.4
-        self.assertEqual(res.headers.get('Content-Type').decode(), ctype)
+        content_type = res.headers.get('Content-Type')
+        self.assertEqual(ctype, encodeutils.safe_decode(content_type))
 
         jsonutils.loads(res.body)
 

nova/tests/unit/api/openstack/compute/test_volumes.py

@@ -18,6 +18,7 @@ import datetime
 
 import mock
 from oslo_serialization import jsonutils
+from oslo_utils import encodeutils
 from six.moves import urllib
 import webob
 from webob import exc
@@ -319,7 +320,8 @@ class VolumeApiTestV21(test.NoDBTestCase):
         req = fakes.HTTPRequest.blank(self.url_prefix + '/os-volumes/456')
         resp = req.get_response(self.app)
         self.assertEqual(404, resp.status_int)
-        self.assertIn('Volume 456 could not be found.', resp.body)
+        self.assertIn('Volume 456 could not be found.',
+                      encodeutils.safe_decode(resp.body))
 
     def test_volume_delete(self):
         req = fakes.HTTPRequest.blank(self.url_prefix + '/os-volumes/123')
@@ -334,7 +336,8 @@ class VolumeApiTestV21(test.NoDBTestCase):
         req.method = 'DELETE'
         resp = req.get_response(self.app)
         self.assertEqual(404, resp.status_int)
-        self.assertIn('Volume 456 could not be found.', resp.body)
+        self.assertIn('Volume 456 could not be found.',
+                      encodeutils.safe_decode(resp.body))
 
 
 class VolumeAttachTestsV21(test.NoDBTestCase):

nova/tests/unit/api/openstack/test_wsgi.py

@@ -867,9 +867,9 @@ class ResourceTest(MicroversionedTest):
         for val in six.itervalues(response.headers):
             # All headers must be utf8
             self.assertThat(val, matchers.EncodedByUTF8())
-        self.assertEqual(b'1', response.headers['x-header1'])
-        self.assertEqual(b'header2', response.headers['x-header2'])
-        self.assertEqual(b'header3', response.headers['x-header3'])
+        self.assertEqual('1', response.headers['x-header1'])
+        self.assertEqual('header2', response.headers['x-header2'])
+        self.assertEqual('header3', response.headers['x-header3'])
 
     def test_resource_valid_utf8_body(self):
         class Controller(object):

nova/tests/unit/matchers.py

@@ -547,6 +547,12 @@ class EncodedByUTF8(object):
                 except UnicodeDecodeError:
                     return testtools.matchers.Mismatch(
                         "%s is not encoded in UTF-8." % obj)
+        elif isinstance(obj, six.text_type):
+            try:
+                obj.encode("utf-8", "strict")
+            except UnicodeDecodeError:
+                return testtools.matchers.Mismatch(
+                        "%s cannot be encoded in UTF-8." % obj)
         else:
             reason = ("Type of '%(obj)s' is '%(obj_type)s', "
                       "should be '%(correct_type)s'."

tests-py3.txt

@@ -29,7 +29,6 @@ nova.tests.unit.api.openstack.compute.test_user_data.ServersControllerCreateTest
 nova.tests.unit.api.openstack.compute.test_versions.VersionsTestV21
 nova.tests.unit.api.openstack.compute.test_versions.VersionsTestV21WithV2CompatibleWrapper
 nova.tests.unit.api.openstack.compute.test_volumes.BootFromVolumeTest
-nova.tests.unit.api.openstack.compute.test_volumes.VolumeApiTestV21
 nova.tests.unit.api.test_compute_req_id.RequestIdTest
 nova.tests.unit.compute.test_compute.ComputeAPITestCase.test_create_with_base64_user_data
 nova.tests.unit.compute.test_compute_cells.CellsComputeAPITestCase.test_create_with_base64_user_data