From 5c62630a9d5142c391eff1117dd86c6a25f20254 Mon Sep 17 00:00:00 2001 From: Gary Kotton Date: Mon, 30 Dec 2013 07:37:35 -0800 Subject: [PATCH] Ensure that nova client prints dictionaries and arrays correctly Print the dictionaries and arrays without the unicode tags. The patch also updated tests that did not return valid data. Change-Id: Ia787f98a9510b68beb3ceaf00c285ca5c934f5c0 Closes-bug: #1265002 --- novaclient/openstack/common/importutils.py | 66 ++++++++ novaclient/openstack/common/jsonutils.py | 182 +++++++++++++++++++++ novaclient/tests/test_utils.py | 33 ++++ novaclient/tests/v1_1/fakes.py | 4 +- novaclient/utils.py | 11 +- openstack-common.conf | 1 + 6 files changed, 292 insertions(+), 5 deletions(-) create mode 100644 novaclient/openstack/common/importutils.py create mode 100644 novaclient/openstack/common/jsonutils.py diff --git a/novaclient/openstack/common/importutils.py b/novaclient/openstack/common/importutils.py new file mode 100644 index 000000000..4fd9ae2bc --- /dev/null +++ b/novaclient/openstack/common/importutils.py @@ -0,0 +1,66 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Import related utilities and helper functions. +""" + +import sys +import traceback + + +def import_class(import_str): + """Returns a class from a string including module and class.""" + mod_str, _sep, class_str = import_str.rpartition('.') + try: + __import__(mod_str) + return getattr(sys.modules[mod_str], class_str) + except (ValueError, AttributeError): + raise ImportError('Class %s cannot be found (%s)' % + (class_str, + traceback.format_exception(*sys.exc_info()))) + + +def import_object(import_str, *args, **kwargs): + """Import a class and return an instance of it.""" + return import_class(import_str)(*args, **kwargs) + + +def import_object_ns(name_space, import_str, *args, **kwargs): + """Tries to import object from default namespace. + + Imports a class and return an instance of it, first by trying + to find the class in a default namespace, then failing back to + a full path if not found in the default namespace. + """ + import_value = "%s.%s" % (name_space, import_str) + try: + return import_class(import_value)(*args, **kwargs) + except ImportError: + return import_class(import_str)(*args, **kwargs) + + +def import_module(import_str): + """Import a module.""" + __import__(import_str) + return sys.modules[import_str] + + +def try_import(import_str, default=None): + """Try to import a module and if it fails return default.""" + try: + return import_module(import_str) + except ImportError: + return default diff --git a/novaclient/openstack/common/jsonutils.py b/novaclient/openstack/common/jsonutils.py new file mode 100644 index 000000000..5595d0b2e --- /dev/null +++ b/novaclient/openstack/common/jsonutils.py @@ -0,0 +1,182 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +''' +JSON related utilities. + +This module provides a few things: + + 1) A handy function for getting an object down to something that can be + JSON serialized. See to_primitive(). + + 2) Wrappers around loads() and dumps(). The dumps() wrapper will + automatically use to_primitive() for you if needed. + + 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson + is available. +''' + + +import datetime +import functools +import inspect +import itertools +import json +try: + import xmlrpclib +except ImportError: + # NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3 + # however the function and object call signatures + # remained the same. This whole try/except block should + # be removed and replaced with a call to six.moves once + # six 1.4.2 is released. See http://bit.ly/1bqrVzu + import xmlrpc.client as xmlrpclib + +import six + +from novaclient.openstack.common import gettextutils +from novaclient.openstack.common import importutils +from novaclient.openstack.common import timeutils + +netaddr = importutils.try_import("netaddr") + +_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod, + inspect.isfunction, inspect.isgeneratorfunction, + inspect.isgenerator, inspect.istraceback, inspect.isframe, + inspect.iscode, inspect.isbuiltin, inspect.isroutine, + inspect.isabstract] + +_simple_types = (six.string_types + six.integer_types + + (type(None), bool, float)) + + +def to_primitive(value, convert_instances=False, convert_datetime=True, + level=0, max_depth=3): + """Convert a complex object into primitives. + + Handy for JSON serialization. We can optionally handle instances, + but since this is a recursive function, we could have cyclical + data structures. + + To handle cyclical data structures we could track the actual objects + visited in a set, but not all objects are hashable. Instead we just + track the depth of the object inspections and don't go too deep. + + Therefore, convert_instances=True is lossy ... be aware. + + """ + # handle obvious types first - order of basic types determined by running + # full tests on nova project, resulting in the following counts: + # 572754 + # 460353 + # 379632 + # 274610 + # 199918 + # 114200 + # 51817 + # 26164 + # 6491 + # 283 + # 19 + if isinstance(value, _simple_types): + return value + + if isinstance(value, datetime.datetime): + if convert_datetime: + return timeutils.strtime(value) + else: + return value + + # value of itertools.count doesn't get caught by nasty_type_tests + # and results in infinite loop when list(value) is called. + if type(value) == itertools.count: + return six.text_type(value) + + # FIXME(vish): Workaround for LP bug 852095. Without this workaround, + # tests that raise an exception in a mocked method that + # has a @wrap_exception with a notifier will fail. If + # we up the dependency to 0.5.4 (when it is released) we + # can remove this workaround. + if getattr(value, '__module__', None) == 'mox': + return 'mock' + + if level > max_depth: + return '?' + + # The try block may not be necessary after the class check above, + # but just in case ... + try: + recursive = functools.partial(to_primitive, + convert_instances=convert_instances, + convert_datetime=convert_datetime, + level=level, + max_depth=max_depth) + if isinstance(value, dict): + return dict((k, recursive(v)) for k, v in six.iteritems(value)) + elif isinstance(value, (list, tuple)): + return [recursive(lv) for lv in value] + + # It's not clear why xmlrpclib created their own DateTime type, but + # for our purposes, make it a datetime type which is explicitly + # handled + if isinstance(value, xmlrpclib.DateTime): + value = datetime.datetime(*tuple(value.timetuple())[:6]) + + if convert_datetime and isinstance(value, datetime.datetime): + return timeutils.strtime(value) + elif isinstance(value, gettextutils.Message): + return value.data + elif hasattr(value, 'iteritems'): + return recursive(dict(value.iteritems()), level=level + 1) + elif hasattr(value, '__iter__'): + return recursive(list(value)) + elif convert_instances and hasattr(value, '__dict__'): + # Likely an instance of something. Watch for cycles. + # Ignore class member vars. + return recursive(value.__dict__, level=level + 1) + elif netaddr and isinstance(value, netaddr.IPAddress): + return six.text_type(value) + else: + if any(test(value) for test in _nasty_type_tests): + return six.text_type(value) + return value + except TypeError: + # Class objects are tricky since they may define something like + # __iter__ defined but it isn't callable as list(). + return six.text_type(value) + + +def dumps(value, default=to_primitive, **kwargs): + return json.dumps(value, default=default, **kwargs) + + +def loads(s): + return json.loads(s) + + +def load(s): + return json.load(s) + + +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append((__name__, 'dumps', TypeError, + 'loads', ValueError, 'load')) + anyjson.force_implementation(__name__) diff --git a/novaclient/tests/test_utils.py b/novaclient/tests/test_utils.py index bc63c95c9..d85175591 100644 --- a/novaclient/tests/test_utils.py +++ b/novaclient/tests/test_utils.py @@ -177,6 +177,39 @@ class PrintResultTestCase(test_utils.TestCase): '| k2 | 2 |\n' '+------+-------+\n') + @mock.patch('sys.stdout', six.StringIO()) + def test_print_dict_dictionary(self): + dict = {'k': {'foo': 'bar'}} + utils.print_dict(dict) + self.assertEqual(sys.stdout.getvalue(), + '+----------+----------------+\n' + '| Property | Value |\n' + '+----------+----------------+\n' + '| k | {"foo": "bar"} |\n' + '+----------+----------------+\n') + + @mock.patch('sys.stdout', six.StringIO()) + def test_print_dict_list_dictionary(self): + dict = {'k': [{'foo': 'bar'}]} + utils.print_dict(dict) + self.assertEqual(sys.stdout.getvalue(), + '+----------+------------------+\n' + '| Property | Value |\n' + '+----------+------------------+\n' + '| k | [{"foo": "bar"}] |\n' + '+----------+------------------+\n') + + @mock.patch('sys.stdout', six.StringIO()) + def test_print_dict_list(self): + dict = {'k': ['foo', 'bar']} + utils.print_dict(dict) + self.assertEqual(sys.stdout.getvalue(), + '+----------+----------------+\n' + '| Property | Value |\n' + '+----------+----------------+\n' + '| k | ["foo", "bar"] |\n' + '+----------+----------------+\n') + class FlattenTestCase(test_utils.TestCase): def test_flattening(self): diff --git a/novaclient/tests/v1_1/fakes.py b/novaclient/tests/v1_1/fakes.py index a6699c0b1..9dbf8ebff 100644 --- a/novaclient/tests/v1_1/fakes.py +++ b/novaclient/tests/v1_1/fakes.py @@ -1944,7 +1944,7 @@ class FakeHTTPClient(base_client.HTTPClient): 'username': 'cell1_user', 'name': 'cell1', 'rpc_host': '10.0.1.10', - '_info': { + 'info': { 'username': 'cell1_user', 'rpc_host': '10.0.1.10', 'type': 'child', @@ -1953,7 +1953,7 @@ class FakeHTTPClient(base_client.HTTPClient): }, 'type': 'child', 'rpc_port': 5673, - '_loaded': True + 'loaded': True }} return (200, {}, cell) diff --git a/novaclient/utils.py b/novaclient/utils.py index 6e6406037..39e713ca5 100644 --- a/novaclient/utils.py +++ b/novaclient/utils.py @@ -23,6 +23,7 @@ import prettytable import six from novaclient import exceptions +from novaclient.openstack.common import jsonutils from novaclient.openstack.common import strutils @@ -237,8 +238,8 @@ def print_dict(d, dict_property="Property", dict_value="Value", wrap=0): pt.align = 'l' for k, v in sorted(d.items()): # convert dict to str to check length - if isinstance(v, dict): - v = str(v) + if isinstance(v, (dict, list)): + v = jsonutils.dumps(v) if wrap > 0: v = textwrap.fill(str(v), wrap) # if value has a newline, add in multiple rows @@ -251,7 +252,11 @@ def print_dict(d, dict_property="Property", dict_value="Value", wrap=0): col1 = '' else: pt.add_row([k, v]) - print(strutils.safe_encode(pt.get_string())) + + result = strutils.safe_encode(pt.get_string()) + if six.PY3: + result = result.decode() + print(result) def find_resource(manager, name_or_id, **find_args): diff --git a/openstack-common.conf b/openstack-common.conf index 8b60c5b5d..e7fe529f1 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -2,6 +2,7 @@ # The list of modules to copy from openstack-common module=install_venv_common +module=jsonutils module=strutils module=timeutils module=uuidutils