From e6e87aa55eca3c6495d75aadc68492031a8ebcf2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 6 Oct 2015 16:43:39 +0200 Subject: [PATCH] py3: Fix unicode versus bytes issues * DataTable: don't try to decode Unicode from UTF-8. Only decode on Python 2. * Use oslo_serialization.jsonutils.dump_as_bytes() to encode JSON as bytes. * Use oslo_serialization.jsonutils.loads() instead of json.loads() to accept bytes string as input. On Python 3, json.loads() only accepts Unicode string. * Fix unit tests: HTTP body type is bytes. * Container tests: use a named temporary file, the function handling the filename fails. On Python 3, TemporaryFile() uses a number for the filename: the file descriptor. * Don't encode filename to UTF-8 on Python 3, keep Unicode. * tox.ini: add 2 admin tests on Python 3.4. Partial-Implements: blueprint porting-python3 Change-Id: Ib7e2cb17f20474590fac18faf8116131692ad694 --- horizon/tables/base.py | 8 ++++++-- .../data_processing/cluster_templates/tests.py | 11 ++++++----- .../content/data_processing/clusters/tests.py | 5 ++--- .../dashboards/admin/aggregates/tests.py | 2 +- .../dashboards/admin/metering/tests.py | 3 +-- .../dashboards/project/containers/tests.py | 13 +++++++------ .../dashboards/project/containers/views.py | 5 ++++- openstack_dashboard/test/test_data/swift_data.py | 2 +- tox.ini | 2 ++ 9 files changed, 30 insertions(+), 21 deletions(-) diff --git a/horizon/tables/base.py b/horizon/tables/base.py index 59f01c374c..e8bad0dd1d 100644 --- a/horizon/tables/base.py +++ b/horizon/tables/base.py @@ -1323,12 +1323,16 @@ class DataTable(object): Uses :meth:`~horizon.tables.DataTable.get_object_id` internally. """ if not isinstance(lookup, six.text_type): - lookup = six.text_type(str(lookup), 'utf-8') + lookup = str(lookup) + if six.PY2: + lookup = lookup.decode('utf-8') matches = [] for datum in self.data: obj_id = self.get_object_id(datum) if not isinstance(obj_id, six.text_type): - obj_id = six.text_type(str(obj_id), 'utf-8') + obj_id = str(obj_id) + if six.PY2: + obj_id = obj_id.decode('utf-8') if obj_id == lookup: matches.append(datum) if len(matches) > 1: diff --git a/openstack_dashboard/contrib/sahara/content/data_processing/cluster_templates/tests.py b/openstack_dashboard/contrib/sahara/content/data_processing/cluster_templates/tests.py index 893204a7fa..a63e884fd2 100644 --- a/openstack_dashboard/contrib/sahara/content/data_processing/cluster_templates/tests.py +++ b/openstack_dashboard/contrib/sahara/content/data_processing/cluster_templates/tests.py @@ -12,12 +12,12 @@ import base64 import copy -import json from django.core.urlresolvers import reverse from django import http from mox3.mox import IsA # noqa +from oslo_serialization import jsonutils import six from openstack_dashboard import api as dash_api @@ -141,6 +141,9 @@ class DataProcessingClusterTemplateTests(test.TestCase): url = reverse('horizon:project:data_processing.cluster_templates:edit', args=[ct.id]) + def serialize(obj): + return base64.urlsafe_b64encode(jsonutils.dump_as_bytes(obj)) + res = self.client.post( url, {'ct_id': ct.id, @@ -152,13 +155,11 @@ class DataProcessingClusterTemplateTests(test.TestCase): 'template_id_0': ct.node_groups[0]['node_group_template_id'], 'group_name_0': ct.node_groups[0]['name'], 'count_0': 1, - 'serialized_0': base64.urlsafe_b64encode( - json.dumps(ct.node_groups[0])), + 'serialized_0': serialize(ct.node_groups[0]), 'template_id_1': ct.node_groups[1]['node_group_template_id'], 'group_name_1': ct.node_groups[1]['name'], 'count_1': 2, - 'serialized_1': base64.urlsafe_b64encode( - json.dumps(ct.node_groups[1])), + 'serialized_1': serialize(ct.node_groups[1]), 'forms_ids': "[0,1]", 'anti-affinity': ct.anti_affinity, }) diff --git a/openstack_dashboard/contrib/sahara/content/data_processing/clusters/tests.py b/openstack_dashboard/contrib/sahara/content/data_processing/clusters/tests.py index dd7e1fc304..95214c50b7 100644 --- a/openstack_dashboard/contrib/sahara/content/data_processing/clusters/tests.py +++ b/openstack_dashboard/contrib/sahara/content/data_processing/clusters/tests.py @@ -10,12 +10,11 @@ # License for the specific language governing permissions and limitations # under the License. -import json - from django.core.urlresolvers import reverse from django import http from mox3.mox import IsA # noqa +from oslo_serialization import jsonutils from openstack_dashboard.contrib.sahara import api from openstack_dashboard.test import helpers as test @@ -61,7 +60,7 @@ class DataProcessingClusterTests(test.TestCase): url = reverse( 'horizon:project:data_processing.clusters:events', args=["cl2"]) res = self.client.get(url) - data = json.loads(res.content) + data = jsonutils.loads(res.content) self.assertIn("provision_steps", data) self.assertEqual(data["need_update"], False) diff --git a/openstack_dashboard/dashboards/admin/aggregates/tests.py b/openstack_dashboard/dashboards/admin/aggregates/tests.py index 84718c1dac..8941679a70 100644 --- a/openstack_dashboard/dashboards/admin/aggregates/tests.py +++ b/openstack_dashboard/dashboards/admin/aggregates/tests.py @@ -188,7 +188,7 @@ class AggregatesViewTests(test.BaseAdminViewTests): self.patchers['aggregates'].stop() res = self.client.get(reverse('horizon:admin:overview:index')) - self.assertNotIn('Host Aggregates', res.content) + self.assertNotIn(b'Host Aggregates', res.content) @test.create_stubs({api.nova: ('aggregate_details_list', 'availability_zone_list',)}) diff --git a/openstack_dashboard/dashboards/admin/metering/tests.py b/openstack_dashboard/dashboards/admin/metering/tests.py index 5447f5eda2..aca34a2608 100644 --- a/openstack_dashboard/dashboards/admin/metering/tests.py +++ b/openstack_dashboard/dashboards/admin/metering/tests.py @@ -14,9 +14,8 @@ from django.core.urlresolvers import reverse from django import http from mox3.mox import IsA # noqa -import six - from oslo_serialization import jsonutils +import six from openstack_dashboard import api from openstack_dashboard.test import helpers as test diff --git a/openstack_dashboard/dashboards/project/containers/tests.py b/openstack_dashboard/dashboards/project/containers/tests.py index 44cd999b8c..a85fdb31be 100644 --- a/openstack_dashboard/dashboards/project/containers/tests.py +++ b/openstack_dashboard/dashboards/project/containers/tests.py @@ -200,9 +200,9 @@ class SwiftTests(test.TestCase): def test_upload(self): container = self.containers.first() obj = self.objects.first() - OBJECT_DATA = 'objectData' + OBJECT_DATA = b'objectData' - temp_file = tempfile.TemporaryFile() + temp_file = tempfile.NamedTemporaryFile() temp_file.write(OBJECT_DATA) temp_file.flush() temp_file.seek(0) @@ -367,8 +367,9 @@ class SwiftTests(test.TestCase): # Check that the returned Content-Disposition filename is well # surrounded by double quotes and with commas removed - expected_name = '"%s"' % obj.name.replace( - ',', '').encode('utf-8') + expected_name = '"%s"' % obj.name.replace(',', '') + if six.PY2: + expected_name = expected_name.encode('utf-8') self.assertEqual( res.get('Content-Disposition'), 'attachment; filename=%s' % expected_name @@ -444,9 +445,9 @@ class SwiftTests(test.TestCase): def test_update_with_file(self): container = self.containers.first() obj = self.objects.first() - OBJECT_DATA = 'objectData' + OBJECT_DATA = b'objectData' - temp_file = tempfile.TemporaryFile() + temp_file = tempfile.NamedTemporaryFile() temp_file.write(OBJECT_DATA) temp_file.flush() temp_file.seek(0) diff --git a/openstack_dashboard/dashboards/project/containers/views.py b/openstack_dashboard/dashboards/project/containers/views.py index f7b1801e2f..78021ee486 100644 --- a/openstack_dashboard/dashboards/project/containers/views.py +++ b/openstack_dashboard/dashboards/project/containers/views.py @@ -26,6 +26,7 @@ from django import http from django.utils.functional import cached_property # noqa from django.utils.translation import ugettext_lazy as _ from django.views import generic +import six from horizon import browsers from horizon import exceptions @@ -207,7 +208,9 @@ def object_download(request, container_name, object_path): name, ext = os.path.splitext(obj.orig_name) filename = "%s%s" % (filename, ext) response = http.StreamingHttpResponse(obj.data) - safe_name = filename.replace(",", "").encode('utf-8') + safe_name = filename.replace(",", "") + if six.PY2: + safe_name = safe_name.encode('utf-8') response['Content-Disposition'] = 'attachment; filename="%s"' % safe_name response['Content-Type'] = 'application/octet-stream' response['Content-Length'] = obj.bytes diff --git a/openstack_dashboard/test/test_data/swift_data.py b/openstack_dashboard/test/test_data/swift_data.py index daa8167f01..391db286c1 100644 --- a/openstack_dashboard/test/test_data/swift_data.py +++ b/openstack_dashboard/test/test_data/swift_data.py @@ -80,7 +80,7 @@ def data(TEST): "last_modified": None, "hash": u"object_hash"} obj_dicts = [object_dict, object_dict_2, object_dict_3, object_dict_4] - obj_data = "Fake Data" + obj_data = b"Fake Data" for obj_dict in obj_dicts: swift_object = swift.StorageObject(obj_dict, diff --git a/tox.ini b/tox.ini index dae15fba86..6a0f87958a 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,8 @@ commands = openstack_dashboard.contrib.sahara.content.data_processing.job_binaries.tests \ openstack_dashboard.contrib.sahara.content.data_processing.jobs.tests \ openstack_dashboard.dashboards.admin.volumes.volumes.tests \ + openstack_dashboard.dashboards.admin.aggregates \ + openstack_dashboard.dashboards.admin.metering \ openstack_dashboard.dashboards.identity.users \ openstack_dashboard.dashboards.project.access_and_security.api_access.tests \ openstack_dashboard.dashboards.project.images.images.tests.CreateImageFormTests \