x/gantt
Code Issues Proposed changes

Make use of openstack.common.jsonutils.

Browse Source 
This patch imports jsonutils from openstack-common.  It removes the
equivalent code from nova.utils and then converts the code base to use
jsonutils.  The primary motivator for this change was to remove the rest
of the dependencies from nova.rpc on nova.utils.

Change-Id: If43658b9b098ed56cba018c81be268b8c3e2916a
This commit is contained in:
Russell Bryant
2012-05-15 12:54:16 -04:00
parent e29420c92f
commit 095209d0a1
9 changed files with 151 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
nova/notifier/api.py
View File

@@ -21,6 +21,7 @@ from nova import flags
from nova import log as logging
from nova.openstack.common import cfg
from nova.openstack.common import importutils
from nova.openstack.common import jsonutils
from nova import utils
@@ -121,7 +122,7 @@ def notify(context, publisher_id, event_type, priority, payload):
_('%s not in valid priorities') % priority)
# Ensure everything is JSON serializable.
payload = utils.to_primitive(payload, convert_instances=True)
payload = jsonutils.to_primitive(payload, convert_instances=True)
driver = importutils.import_module(FLAGS.notification_driver)
msg = dict(message_id=str(uuid.uuid4()),

133
nova/openstack/common/jsonutils.py Normal file
View File

@@ -0,0 +1,133 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2011 Justin Santa Barbara
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
'''
JSON related utilities.
This module provides a few things:
1) A handy function for getting an object down to something that can be
JSON serialized. See to_primitive().
2) Wrappers around loads() and dumps(). The dumps() wrapper will
automatically use to_primitive() for you if needed.
3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
is available.
'''
import datetime
import inspect
import itertools
import json
def to_primitive(value, convert_instances=False, level=0):
"""Convert a complex object into primitives.
Handy for JSON serialization. We can optionally handle instances,
but since this is a recursive function, we could have cyclical
data structures.
To handle cyclical data structures we could track the actual objects
visited in a set, but not all objects are hashable. Instead we just
track the depth of the object inspections and don't go too deep.
Therefore, convert_instances=True is lossy ... be aware.
"""
nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod,
inspect.isfunction, inspect.isgeneratorfunction,
inspect.isgenerator, inspect.istraceback, inspect.isframe,
inspect.iscode, inspect.isbuiltin, inspect.isroutine,
inspect.isabstract]
for test in nasty:
if test(value):
return unicode(value)
# value of itertools.count doesn't get caught by inspects
# above and results in infinite loop when list(value) is called.
if type(value) == itertools.count:
return unicode(value)
# FIXME(vish): Workaround for LP bug 852095. Without this workaround,
# tests that raise an exception in a mocked method that
# has a @wrap_exception with a notifier will fail. If
# we up the dependency to 0.5.4 (when it is released) we
# can remove this workaround.
if getattr(value, '__module__', None) == 'mox':
return 'mock'
if level > 3:
return '?'
# The try block may not be necessary after the class check above,
# but just in case ...
try:
if isinstance(value, (list, tuple)):
o = []
for v in value:
o.append(to_primitive(v, convert_instances=convert_instances,
level=level))
return o
elif isinstance(value, dict):
o = {}
for k, v in value.iteritems():
o[k] = to_primitive(v, convert_instances=convert_instances,
level=level)
return o
elif isinstance(value, datetime.datetime):
return str(value)
elif hasattr(value, 'iteritems'):
return to_primitive(dict(value.iteritems()),
convert_instances=convert_instances,
level=level)
elif hasattr(value, '__iter__'):
return to_primitive(list(value), level)
elif convert_instances and hasattr(value, '__dict__'):
# Likely an instance of something. Watch for cycles.
# Ignore class member vars.
return to_primitive(value.__dict__,
convert_instances=convert_instances,
level=level + 1)
else:
return value
except TypeError, e:
# Class objects are tricky since they may define something like
# __iter__ defined but it isn't callable as list().
return unicode(value)
def dumps(value):
return json.dumps(value, default=to_primitive)
def loads(s):
return json.loads(s)
try:
import anyjson
except ImportError:
pass
else:
anyjson._modules.append((__name__, 'dumps', TypeError,
'loads', ValueError))
anyjson.force_implementation(__name__)

6
nova/rpc/common.py
View File

@@ -25,7 +25,7 @@ from nova import exception
from nova import log as logging
from nova.openstack.common import cfg
from nova.openstack.common import importutils
from nova import utils
from nova.openstack.common import jsonutils
LOG = logging.getLogger(__name__)
@@ -174,13 +174,13 @@ def serialize_remote_exception(failure_info):
'kwargs': kwargs
}
json_data = utils.dumps(data)
json_data = jsonutils.dumps(data)
return json_data
def deserialize_remote_exception(conf, data):
failure = utils.loads(str(data))
failure = jsonutils.loads(str(data))
trace = failure.get('tb', [])
message = failure.get('message', "") + "\n" + "\n".join(trace)

3
nova/scheduler/driver.py
View File

@@ -30,6 +30,7 @@ from nova import flags
from nova import log as logging
from nova.openstack.common import cfg
from nova.openstack.common import importutils
from nova.openstack.common import jsonutils
from nova import rpc
from nova.rpc import common as rpc_common
from nova import utils
@@ -442,7 +443,7 @@ class Scheduler(object):
ret = rpc.call(context, topic,
{"method": 'get_instance_disk_info',
"args": {'instance_name': instance_ref['name']}})
disk_infos = utils.loads(ret)
disk_infos = jsonutils.loads(ret)
necessary = 0
if disk_over_commit:

6
nova/tests/baremetal/test_proxy_bare_metal.py
View File

@@ -20,9 +20,9 @@ import mox
import StringIO
from nova import flags
from nova import utils
from nova import test
from nova.compute import power_state
from nova.openstack.common import jsonutils
from nova.tests import fake_utils
from nova import exception
@@ -95,8 +95,8 @@ class DomainReadWriteTestCase(test.TestCase):
def assertJSONEquals(self, x, y):
"""Check if two json strings represent the equivalent Python object"""
self.assertEquals(utils.loads(x), utils.loads(y))
return utils.loads(x) == utils.loads(y)
self.assertEquals(jsonutils.loads(x), jsonutils.loads(y))
return jsonutils.loads(x) == jsonutils.loads(y)
def test_write_domain(self):
"""Write the domain to file"""

4
nova/tests/notifier/test_capacity_notifier.py
View File

@@ -15,8 +15,8 @@
import nova.db.api
from nova.notifier import capacity_notifier as cn
from nova.openstack.common import jsonutils
from nova import test
from nova import utils
class CapacityNotifierTestCase(test.TestCase):
@@ -24,7 +24,7 @@ class CapacityNotifierTestCase(test.TestCase):
def _make_msg(self, host, event):
usage_info = dict(memory_mb=123, disk_gb=456)
payload = utils.to_primitive(usage_info, convert_instances=True)
payload = jsonutils.to_primitive(usage_info, convert_instances=True)
return dict(
publisher_id="compute.%s" % host,
event_type="compute.instance.%s" % event,

9
nova/tests/test_libvirt.py
View File

@@ -38,6 +38,7 @@ from nova import exception
from nova import flags
from nova import log as logging
from nova.openstack.common import importutils
from nova.openstack.common import jsonutils
from nova import test
from nova.tests import fake_network
from nova.tests import fake_libvirt_utils
@@ -1405,7 +1406,7 @@ class LibvirtConnTestCase(test.TestCase):
self.mox.ReplayAll()
conn = connection.LibvirtConnection(False)
info = conn.get_instance_disk_info(instance_ref.name)
info = utils.loads(info)
info = jsonutils.loads(info)
self.assertEquals(info[0]['type'], 'raw')
self.assertEquals(info[0]['path'], '/test/disk')
self.assertEquals(info[0]['disk_size'], 10737418240)
@@ -2507,7 +2508,7 @@ class LibvirtConnectionTestCase(test.TestCase):
'virt_disk_size': '10737418240',
'backing_file': '/base/disk.local',
'disk_size':'83886080'}]
disk_info_text = utils.dumps(disk_info)
disk_info_text = jsonutils.dumps(disk_info)
def fake_get_instance_disk_info(instance):
return disk_info_text
@@ -2578,7 +2579,7 @@ class LibvirtConnectionTestCase(test.TestCase):
'local_gb': 10, 'backing_file': '/base/disk'},
{'type': 'raw', 'path': '/test/disk.local',
'local_gb': 10, 'backing_file': '/base/disk.local'}]
disk_info_text = utils.dumps(disk_info)
disk_info_text = jsonutils.dumps(disk_info)
def fake_extend(path, size):
pass
@@ -2668,4 +2669,4 @@ class LibvirtNonblockingTestCase(test.TestCase):
"""Test bug 962840"""
import nova.virt.libvirt.connection
connection = nova.virt.libvirt.connection.get_connection('')
utils.to_primitive(connection._conn, convert_instances=True)
jsonutils.to_primitive(connection._conn, convert_instances=True)

100
nova/utils.py
View File

@@ -25,8 +25,6 @@ import errno
import functools
import hashlib
import inspect
import itertools
import json
import os
import pyclbr
import random
@@ -677,104 +675,6 @@ def utf8(value):
return value
def to_primitive(value, convert_instances=False, level=0):
"""Convert a complex object into primitives.
Handy for JSON serialization. We can optionally handle instances,
but since this is a recursive function, we could have cyclical
data structures.
To handle cyclical data structures we could track the actual objects
visited in a set, but not all objects are hashable. Instead we just
track the depth of the object inspections and don't go too deep.
Therefore, convert_instances=True is lossy ... be aware.
"""
nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod,
inspect.isfunction, inspect.isgeneratorfunction,
inspect.isgenerator, inspect.istraceback, inspect.isframe,
inspect.iscode, inspect.isbuiltin, inspect.isroutine,
inspect.isabstract]
for test in nasty:
if test(value):
return unicode(value)
# value of itertools.count doesn't get caught by inspects
# above and results in infinite loop when list(value) is called.
if type(value) == itertools.count:
return unicode(value)
# FIXME(vish): Workaround for LP bug 852095. Without this workaround,
# tests that raise an exception in a mocked method that
# has a @wrap_exception with a notifier will fail. If
# we up the dependency to 0.5.4 (when it is released) we
# can remove this workaround.
if getattr(value, '__module__', None) == 'mox':
return 'mock'
if level > 3:
return '?'
# The try block may not be necessary after the class check above,
# but just in case ...
try:
if isinstance(value, (list, tuple)):
o = []
for v in value:
o.append(to_primitive(v, convert_instances=convert_instances,
level=level))
return o
elif isinstance(value, dict):
o = {}
for k, v in value.iteritems():
o[k] = to_primitive(v, convert_instances=convert_instances,
level=level)
return o
elif isinstance(value, datetime.datetime):
return str(value)
elif hasattr(value, 'iteritems'):
return to_primitive(dict(value.iteritems()),
convert_instances=convert_instances,
level=level)
elif hasattr(value, '__iter__'):
return to_primitive(list(value), level)
elif convert_instances and hasattr(value, '__dict__'):
# Likely an instance of something. Watch for cycles.
# Ignore class member vars.
return to_primitive(value.__dict__,
convert_instances=convert_instances,
level=level + 1)
else:
return value
except TypeError, e:
# Class objects are tricky since they may define something like
# __iter__ defined but it isn't callable as list().
return unicode(value)
def dumps(value):
try:
return json.dumps(value)
except TypeError:
pass
return json.dumps(to_primitive(value))
def loads(s):
return json.loads(s)
try:
import anyjson
except ImportError:
pass
else:
anyjson._modules.append(("nova.utils", "dumps", TypeError,
"loads", ValueError))
anyjson.force_implementation("nova.utils")
class GreenLockFile(lockfile.FileLock):
"""Implementation of lockfile that allows for a lock per greenthread.

2
openstack-common.conf
View File

@@ -1,7 +1,7 @@
[DEFAULT]
# The list of modules to copy from openstack-common
modules=cfg,excutils,local,importutils,iniparser,setup
modules=cfg,excutils,local,importutils,iniparser,jsonutils,setup
# The base module to hold the copy of openstack.common
base=nova