diff --git a/nova/notifier/api.py b/nova/notifier/api.py index b3bfbf9c4..bc2690077 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -21,6 +21,7 @@ from nova import flags from nova import log as logging from nova.openstack.common import cfg from nova.openstack.common import importutils +from nova.openstack.common import jsonutils from nova import utils @@ -121,7 +122,7 @@ def notify(context, publisher_id, event_type, priority, payload): _('%s not in valid priorities') % priority) # Ensure everything is JSON serializable. - payload = utils.to_primitive(payload, convert_instances=True) + payload = jsonutils.to_primitive(payload, convert_instances=True) driver = importutils.import_module(FLAGS.notification_driver) msg = dict(message_id=str(uuid.uuid4()), diff --git a/nova/openstack/common/jsonutils.py b/nova/openstack/common/jsonutils.py new file mode 100644 index 000000000..fa8b8f9d1 --- /dev/null +++ b/nova/openstack/common/jsonutils.py @@ -0,0 +1,133 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +''' +JSON related utilities. + +This module provides a few things: + + 1) A handy function for getting an object down to something that can be + JSON serialized. See to_primitive(). + + 2) Wrappers around loads() and dumps(). The dumps() wrapper will + automatically use to_primitive() for you if needed. + + 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson + is available. +''' + + +import datetime +import inspect +import itertools +import json + + +def to_primitive(value, convert_instances=False, level=0): + """Convert a complex object into primitives. + + Handy for JSON serialization. We can optionally handle instances, + but since this is a recursive function, we could have cyclical + data structures. + + To handle cyclical data structures we could track the actual objects + visited in a set, but not all objects are hashable. Instead we just + track the depth of the object inspections and don't go too deep. + + Therefore, convert_instances=True is lossy ... be aware. + + """ + nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod, + inspect.isfunction, inspect.isgeneratorfunction, + inspect.isgenerator, inspect.istraceback, inspect.isframe, + inspect.iscode, inspect.isbuiltin, inspect.isroutine, + inspect.isabstract] + for test in nasty: + if test(value): + return unicode(value) + + # value of itertools.count doesn't get caught by inspects + # above and results in infinite loop when list(value) is called. + if type(value) == itertools.count: + return unicode(value) + + # FIXME(vish): Workaround for LP bug 852095. Without this workaround, + # tests that raise an exception in a mocked method that + # has a @wrap_exception with a notifier will fail. If + # we up the dependency to 0.5.4 (when it is released) we + # can remove this workaround. + if getattr(value, '__module__', None) == 'mox': + return 'mock' + + if level > 3: + return '?' + + # The try block may not be necessary after the class check above, + # but just in case ... + try: + if isinstance(value, (list, tuple)): + o = [] + for v in value: + o.append(to_primitive(v, convert_instances=convert_instances, + level=level)) + return o + elif isinstance(value, dict): + o = {} + for k, v in value.iteritems(): + o[k] = to_primitive(v, convert_instances=convert_instances, + level=level) + return o + elif isinstance(value, datetime.datetime): + return str(value) + elif hasattr(value, 'iteritems'): + return to_primitive(dict(value.iteritems()), + convert_instances=convert_instances, + level=level) + elif hasattr(value, '__iter__'): + return to_primitive(list(value), level) + elif convert_instances and hasattr(value, '__dict__'): + # Likely an instance of something. Watch for cycles. + # Ignore class member vars. + return to_primitive(value.__dict__, + convert_instances=convert_instances, + level=level + 1) + else: + return value + except TypeError, e: + # Class objects are tricky since they may define something like + # __iter__ defined but it isn't callable as list(). + return unicode(value) + + +def dumps(value): + return json.dumps(value, default=to_primitive) + + +def loads(s): + return json.loads(s) + + +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append((__name__, 'dumps', TypeError, + 'loads', ValueError)) + anyjson.force_implementation(__name__) diff --git a/nova/rpc/common.py b/nova/rpc/common.py index 33aea9bc3..23a191a9b 100644 --- a/nova/rpc/common.py +++ b/nova/rpc/common.py @@ -25,7 +25,7 @@ from nova import exception from nova import log as logging from nova.openstack.common import cfg from nova.openstack.common import importutils -from nova import utils +from nova.openstack.common import jsonutils LOG = logging.getLogger(__name__) @@ -174,13 +174,13 @@ def serialize_remote_exception(failure_info): 'kwargs': kwargs } - json_data = utils.dumps(data) + json_data = jsonutils.dumps(data) return json_data def deserialize_remote_exception(conf, data): - failure = utils.loads(str(data)) + failure = jsonutils.loads(str(data)) trace = failure.get('tb', []) message = failure.get('message', "") + "\n" + "\n".join(trace) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 3fcef8402..fc705780b 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -30,6 +30,7 @@ from nova import flags from nova import log as logging from nova.openstack.common import cfg from nova.openstack.common import importutils +from nova.openstack.common import jsonutils from nova import rpc from nova.rpc import common as rpc_common from nova import utils @@ -442,7 +443,7 @@ class Scheduler(object): ret = rpc.call(context, topic, {"method": 'get_instance_disk_info', "args": {'instance_name': instance_ref['name']}}) - disk_infos = utils.loads(ret) + disk_infos = jsonutils.loads(ret) necessary = 0 if disk_over_commit: diff --git a/nova/tests/baremetal/test_proxy_bare_metal.py b/nova/tests/baremetal/test_proxy_bare_metal.py index 323043800..128b16cb8 100644 --- a/nova/tests/baremetal/test_proxy_bare_metal.py +++ b/nova/tests/baremetal/test_proxy_bare_metal.py @@ -20,9 +20,9 @@ import mox import StringIO from nova import flags -from nova import utils from nova import test from nova.compute import power_state +from nova.openstack.common import jsonutils from nova.tests import fake_utils from nova import exception @@ -95,8 +95,8 @@ class DomainReadWriteTestCase(test.TestCase): def assertJSONEquals(self, x, y): """Check if two json strings represent the equivalent Python object""" - self.assertEquals(utils.loads(x), utils.loads(y)) - return utils.loads(x) == utils.loads(y) + self.assertEquals(jsonutils.loads(x), jsonutils.loads(y)) + return jsonutils.loads(x) == jsonutils.loads(y) def test_write_domain(self): """Write the domain to file""" diff --git a/nova/tests/notifier/test_capacity_notifier.py b/nova/tests/notifier/test_capacity_notifier.py index e41c52700..3b642a77b 100644 --- a/nova/tests/notifier/test_capacity_notifier.py +++ b/nova/tests/notifier/test_capacity_notifier.py @@ -15,8 +15,8 @@ import nova.db.api from nova.notifier import capacity_notifier as cn +from nova.openstack.common import jsonutils from nova import test -from nova import utils class CapacityNotifierTestCase(test.TestCase): @@ -24,7 +24,7 @@ class CapacityNotifierTestCase(test.TestCase): def _make_msg(self, host, event): usage_info = dict(memory_mb=123, disk_gb=456) - payload = utils.to_primitive(usage_info, convert_instances=True) + payload = jsonutils.to_primitive(usage_info, convert_instances=True) return dict( publisher_id="compute.%s" % host, event_type="compute.instance.%s" % event, diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index b0b165c70..db804acee 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -38,6 +38,7 @@ from nova import exception from nova import flags from nova import log as logging from nova.openstack.common import importutils +from nova.openstack.common import jsonutils from nova import test from nova.tests import fake_network from nova.tests import fake_libvirt_utils @@ -1405,7 +1406,7 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = connection.LibvirtConnection(False) info = conn.get_instance_disk_info(instance_ref.name) - info = utils.loads(info) + info = jsonutils.loads(info) self.assertEquals(info[0]['type'], 'raw') self.assertEquals(info[0]['path'], '/test/disk') self.assertEquals(info[0]['disk_size'], 10737418240) @@ -2507,7 +2508,7 @@ class LibvirtConnectionTestCase(test.TestCase): 'virt_disk_size': '10737418240', 'backing_file': '/base/disk.local', 'disk_size':'83886080'}] - disk_info_text = utils.dumps(disk_info) + disk_info_text = jsonutils.dumps(disk_info) def fake_get_instance_disk_info(instance): return disk_info_text @@ -2578,7 +2579,7 @@ class LibvirtConnectionTestCase(test.TestCase): 'local_gb': 10, 'backing_file': '/base/disk'}, {'type': 'raw', 'path': '/test/disk.local', 'local_gb': 10, 'backing_file': '/base/disk.local'}] - disk_info_text = utils.dumps(disk_info) + disk_info_text = jsonutils.dumps(disk_info) def fake_extend(path, size): pass @@ -2668,4 +2669,4 @@ class LibvirtNonblockingTestCase(test.TestCase): """Test bug 962840""" import nova.virt.libvirt.connection connection = nova.virt.libvirt.connection.get_connection('') - utils.to_primitive(connection._conn, convert_instances=True) + jsonutils.to_primitive(connection._conn, convert_instances=True) diff --git a/nova/utils.py b/nova/utils.py index 2b495d1ff..9577ff269 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -25,8 +25,6 @@ import errno import functools import hashlib import inspect -import itertools -import json import os import pyclbr import random @@ -677,104 +675,6 @@ def utf8(value): return value -def to_primitive(value, convert_instances=False, level=0): - """Convert a complex object into primitives. - - Handy for JSON serialization. We can optionally handle instances, - but since this is a recursive function, we could have cyclical - data structures. - - To handle cyclical data structures we could track the actual objects - visited in a set, but not all objects are hashable. Instead we just - track the depth of the object inspections and don't go too deep. - - Therefore, convert_instances=True is lossy ... be aware. - - """ - nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod, - inspect.isfunction, inspect.isgeneratorfunction, - inspect.isgenerator, inspect.istraceback, inspect.isframe, - inspect.iscode, inspect.isbuiltin, inspect.isroutine, - inspect.isabstract] - for test in nasty: - if test(value): - return unicode(value) - - # value of itertools.count doesn't get caught by inspects - # above and results in infinite loop when list(value) is called. - if type(value) == itertools.count: - return unicode(value) - - # FIXME(vish): Workaround for LP bug 852095. Without this workaround, - # tests that raise an exception in a mocked method that - # has a @wrap_exception with a notifier will fail. If - # we up the dependency to 0.5.4 (when it is released) we - # can remove this workaround. - if getattr(value, '__module__', None) == 'mox': - return 'mock' - - if level > 3: - return '?' - - # The try block may not be necessary after the class check above, - # but just in case ... - try: - if isinstance(value, (list, tuple)): - o = [] - for v in value: - o.append(to_primitive(v, convert_instances=convert_instances, - level=level)) - return o - elif isinstance(value, dict): - o = {} - for k, v in value.iteritems(): - o[k] = to_primitive(v, convert_instances=convert_instances, - level=level) - return o - elif isinstance(value, datetime.datetime): - return str(value) - elif hasattr(value, 'iteritems'): - return to_primitive(dict(value.iteritems()), - convert_instances=convert_instances, - level=level) - elif hasattr(value, '__iter__'): - return to_primitive(list(value), level) - elif convert_instances and hasattr(value, '__dict__'): - # Likely an instance of something. Watch for cycles. - # Ignore class member vars. - return to_primitive(value.__dict__, - convert_instances=convert_instances, - level=level + 1) - else: - return value - except TypeError, e: - # Class objects are tricky since they may define something like - # __iter__ defined but it isn't callable as list(). - return unicode(value) - - -def dumps(value): - try: - return json.dumps(value) - except TypeError: - pass - return json.dumps(to_primitive(value)) - - -def loads(s): - return json.loads(s) - - -try: - import anyjson -except ImportError: - pass -else: - anyjson._modules.append(("nova.utils", "dumps", TypeError, - "loads", ValueError)) - anyjson.force_implementation("nova.utils") - - class GreenLockFile(lockfile.FileLock): """Implementation of lockfile that allows for a lock per greenthread. diff --git a/openstack-common.conf b/openstack-common.conf index 538d4b0c0..61690b895 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,7 +1,7 @@ [DEFAULT] # The list of modules to copy from openstack-common -modules=cfg,excutils,local,importutils,iniparser,setup +modules=cfg,excutils,local,importutils,iniparser,jsonutils,setup # The base module to hold the copy of openstack.common base=nova