py3: Fix unicode versus bytes issues

* DataTable: don't try to decode Unicode from UTF-8. Only decode on
  Python 2.
* Use oslo_serialization.jsonutils.dump_as_bytes() to encode JSON
  as bytes.
* Use oslo_serialization.jsonutils.loads() instead of json.loads() to
  accept bytes string as input. On Python 3, json.loads() only
  accepts Unicode string.
* Fix unit tests: HTTP body type is bytes.
* Container tests: use a named temporary file, the function handling
  the filename fails. On Python 3, TemporaryFile() uses a number for
  the filename: the file descriptor.
* Don't encode filename to UTF-8 on Python 3, keep Unicode.
* tox.ini: add 2 admin tests on Python 3.4.

Partial-Implements: blueprint porting-python3
Change-Id: Ib7e2cb17f20474590fac18faf8116131692ad694

horizon/tables/base.py

@@ -1323,12 +1323,16 @@ class DataTable(object):
         Uses :meth:`~horizon.tables.DataTable.get_object_id` internally.
         """
         if not isinstance(lookup, six.text_type):
-            lookup = six.text_type(str(lookup), 'utf-8')
+            lookup = str(lookup)
+            if six.PY2:
+                lookup = lookup.decode('utf-8')
         matches = []
         for datum in self.data:
             obj_id = self.get_object_id(datum)
             if not isinstance(obj_id, six.text_type):
-                obj_id = six.text_type(str(obj_id), 'utf-8')
+                obj_id = str(obj_id)
+                if six.PY2:
+                    obj_id = obj_id.decode('utf-8')
             if obj_id == lookup:
                 matches.append(datum)
         if len(matches) > 1:

openstack_dashboard/contrib/sahara/content/data_processing/cluster_templates/tests.py

@@ -12,12 +12,12 @@
 
 import base64
 import copy
-import json
 
 from django.core.urlresolvers import reverse
 from django import http
 
 from mox3.mox import IsA  # noqa
+from oslo_serialization import jsonutils
 import six
 
 from openstack_dashboard import api as dash_api
@@ -141,6 +141,9 @@ class DataProcessingClusterTemplateTests(test.TestCase):
         url = reverse('horizon:project:data_processing.cluster_templates:edit',
                       args=[ct.id])
 
+        def serialize(obj):
+            return base64.urlsafe_b64encode(jsonutils.dump_as_bytes(obj))
+
         res = self.client.post(
             url,
             {'ct_id': ct.id,
@@ -152,13 +155,11 @@ class DataProcessingClusterTemplateTests(test.TestCase):
              'template_id_0': ct.node_groups[0]['node_group_template_id'],
              'group_name_0': ct.node_groups[0]['name'],
              'count_0': 1,
-             'serialized_0': base64.urlsafe_b64encode(
+             'serialized_0': serialize(ct.node_groups[0]),
-                 json.dumps(ct.node_groups[0])),
              'template_id_1': ct.node_groups[1]['node_group_template_id'],
              'group_name_1': ct.node_groups[1]['name'],
              'count_1': 2,
-             'serialized_1': base64.urlsafe_b64encode(
+             'serialized_1': serialize(ct.node_groups[1]),
-                 json.dumps(ct.node_groups[1])),
              'forms_ids': "[0,1]",
              'anti-affinity': ct.anti_affinity,
              })

openstack_dashboard/contrib/sahara/content/data_processing/clusters/tests.py

@@ -10,12 +10,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import json
-
 from django.core.urlresolvers import reverse
 from django import http
 
 from mox3.mox import IsA  # noqa
+from oslo_serialization import jsonutils
 
 from openstack_dashboard.contrib.sahara import api
 from openstack_dashboard.test import helpers as test
@@ -61,7 +60,7 @@ class DataProcessingClusterTests(test.TestCase):
         url = reverse(
             'horizon:project:data_processing.clusters:events', args=["cl2"])
         res = self.client.get(url)
-        data = json.loads(res.content)
+        data = jsonutils.loads(res.content)
 
         self.assertIn("provision_steps", data)
         self.assertEqual(data["need_update"], False)

openstack_dashboard/dashboards/admin/aggregates/tests.py

@@ -188,7 +188,7 @@ class AggregatesViewTests(test.BaseAdminViewTests):
 
         self.patchers['aggregates'].stop()
         res = self.client.get(reverse('horizon:admin:overview:index'))
-        self.assertNotIn('Host Aggregates', res.content)
+        self.assertNotIn(b'Host Aggregates', res.content)
 
     @test.create_stubs({api.nova: ('aggregate_details_list',
                                    'availability_zone_list',)})

openstack_dashboard/dashboards/admin/metering/tests.py

@@ -14,9 +14,8 @@ from django.core.urlresolvers import reverse
 from django import http
 
 from mox3.mox import IsA  # noqa
-import six
-
 from oslo_serialization import jsonutils
+import six
 
 from openstack_dashboard import api
 from openstack_dashboard.test import helpers as test

openstack_dashboard/dashboards/project/containers/tests.py

@@ -200,9 +200,9 @@ class SwiftTests(test.TestCase):
     def test_upload(self):
         container = self.containers.first()
         obj = self.objects.first()
-        OBJECT_DATA = 'objectData'
+        OBJECT_DATA = b'objectData'
 
-        temp_file = tempfile.TemporaryFile()
+        temp_file = tempfile.NamedTemporaryFile()
         temp_file.write(OBJECT_DATA)
         temp_file.flush()
         temp_file.seek(0)
@@ -367,8 +367,9 @@ class SwiftTests(test.TestCase):
 
                 # Check that the returned Content-Disposition filename is well
                 # surrounded by double quotes and with commas removed
-                expected_name = '"%s"' % obj.name.replace(
+                expected_name = '"%s"' % obj.name.replace(',', '')
-                    ',', '').encode('utf-8')
+                if six.PY2:
+                    expected_name = expected_name.encode('utf-8')
                 self.assertEqual(
                     res.get('Content-Disposition'),
                     'attachment; filename=%s' % expected_name
@@ -444,9 +445,9 @@ class SwiftTests(test.TestCase):
     def test_update_with_file(self):
         container = self.containers.first()
         obj = self.objects.first()
-        OBJECT_DATA = 'objectData'
+        OBJECT_DATA = b'objectData'
 
-        temp_file = tempfile.TemporaryFile()
+        temp_file = tempfile.NamedTemporaryFile()
         temp_file.write(OBJECT_DATA)
         temp_file.flush()
         temp_file.seek(0)

openstack_dashboard/dashboards/project/containers/views.py

@@ -26,6 +26,7 @@ from django import http
 from django.utils.functional import cached_property  # noqa
 from django.utils.translation import ugettext_lazy as _
 from django.views import generic
+import six
 
 from horizon import browsers
 from horizon import exceptions
@@ -207,7 +208,9 @@ def object_download(request, container_name, object_path):
         name, ext = os.path.splitext(obj.orig_name)
         filename = "%s%s" % (filename, ext)
     response = http.StreamingHttpResponse(obj.data)
-    safe_name = filename.replace(",", "").encode('utf-8')
+    safe_name = filename.replace(",", "")
+    if six.PY2:
+        safe_name = safe_name.encode('utf-8')
     response['Content-Disposition'] = 'attachment; filename="%s"' % safe_name
     response['Content-Type'] = 'application/octet-stream'
     response['Content-Length'] = obj.bytes

openstack_dashboard/test/test_data/swift_data.py

@@ -80,7 +80,7 @@ def data(TEST):
                      "last_modified": None,
                      "hash": u"object_hash"}
     obj_dicts = [object_dict, object_dict_2, object_dict_3, object_dict_4]
-    obj_data = "Fake Data"
+    obj_data = b"Fake Data"
 
     for obj_dict in obj_dicts:
         swift_object = swift.StorageObject(obj_dict,

tox.ini

@@ -28,6 +28,8 @@ commands =
         openstack_dashboard.contrib.sahara.content.data_processing.job_binaries.tests \
         openstack_dashboard.contrib.sahara.content.data_processing.jobs.tests \
         openstack_dashboard.dashboards.admin.volumes.volumes.tests \
+        openstack_dashboard.dashboards.admin.aggregates \
+        openstack_dashboard.dashboards.admin.metering \
         openstack_dashboard.dashboards.identity.users \
         openstack_dashboard.dashboards.project.access_and_security.api_access.tests \
         openstack_dashboard.dashboards.project.images.images.tests.CreateImageFormTests \