Keystone should use openstack.common.jsonutils

Implements blueprint use-common-jsonutils

1. Edit openstack-common.conf and import keystone/openstack/common/jsonutils.py
2. Remove json package imports and replace with jsonutils

Client code in vendor/ hasn't been changed

Change-Id: I57c670fde9f2c2241eddab1b012e8d5e6a72deb7

keystone/cli.py

@@ -16,12 +16,12 @@
 
 from __future__ import absolute_import
 
-import json
 import sys
 import textwrap
 
 from keystone import config
 from keystone.openstack.common import importutils
+from keystone.openstack.common import jsonutils
 
 CONF = config.CONF
 
@@ -102,7 +102,7 @@ class ImportNovaAuth(BaseApp):
         if len(self.argv) < 2:
             return self.missing_param('dump_file')
         dump_file = self.argv[1]
-        dump_data = json.loads(open(dump_file).read())
+        dump_data = jsonutils.loads(open(dump_file).read())
         nova.import_auth(dump_data)
 
 

keystone/common/policy.py

@@ -17,10 +17,11 @@
 
 """Common Policy Engine Implementation"""
 
-import json
 import urllib
 import urllib2
 
+from keystone.openstack.common import jsonutils
+
 
 class NotAuthorized(Exception):
     pass
@@ -109,7 +110,7 @@ class Brain(object):
     @classmethod
     def load_json(cls, data, default_rule=None):
         """Init a brain using json instead of a rules dictionary."""
-        rules_dict = json.loads(data)
+        rules_dict = jsonutils.loads(data)
         return cls(rules=rules_dict, default_rule=default_rule)
 
     def __init__(self, rules=None, default_rule=None):
@@ -203,8 +204,8 @@ class HttpBrain(Brain):
 
         """
         url = match % target_dict
-        data = {'target': json.dumps(target_dict),
-                'credentials': json.dumps(cred_dict)}
+        data = {'target': jsonutils.dumps(target_dict),
+                'credentials': jsonutils.dumps(cred_dict)}
         post_data = urllib.urlencode(data)
         f = urllib2.urlopen(url, post_data)
         return f.read() == "True"

keystone/common/sql/core.py

@@ -16,19 +16,17 @@
 
 """SQL backends for the various services."""
 
-
-import json
-
 import sqlalchemy as sql
-from sqlalchemy import types as sql_types
+import sqlalchemy.engine.url
 from sqlalchemy.exc import DisconnectionError
 from sqlalchemy.ext import declarative
 import sqlalchemy.orm
 import sqlalchemy.pool
-import sqlalchemy.engine.url
+from sqlalchemy import types as sql_types
 
-from keystone import config
 from keystone.common import logging
+from keystone import config
+from keystone.openstack.common import jsonutils
 
 
 CONF = config.CONF
@@ -51,10 +49,10 @@ class JsonBlob(sql_types.TypeDecorator):
     impl = sql.Text
 
     def process_bind_param(self, value, dialect):
-        return json.dumps(value)
+        return jsonutils.dumps(value)
 
     def process_result_value(self, value, dialect):
-        return json.loads(value)
+        return jsonutils.loads(value)
 
 
 class DictBase(object):

keystone/common/wsgi.py

@@ -20,7 +20,6 @@
 
 """Utility methods for working with WSGI servers."""
 
-import json
 import sys
 
 import eventlet.wsgi
@@ -33,6 +32,7 @@ import webob.exc
 from keystone.common import logging
 from keystone.common import utils
 from keystone import exception
+from keystone.openstack.common import jsonutils
 
 
 LOG = logging.getLogger(__name__)
@@ -499,7 +499,7 @@ def render_response(body=None, status=(200, 'OK'), headers=None):
                                   ('Vary', 'X-Auth-Token')]
 
     if body is not None:
-        resp.body = json.dumps(body, cls=utils.SmarterEncoder)
+        resp.body = jsonutils.dumps(body, cls=utils.SmarterEncoder)
 
     return resp
 

keystone/middleware/auth_token.py

@@ -94,13 +94,14 @@ HTTP_X_ROLE
 """
 
 import httplib
-import json
 import logging
 import time
 
 import webob
 import webob.exc
 
+from keystone.openstack.common import jsonutils
+
 
 LOG = logging.getLogger(__name__)
 
@@ -293,7 +294,7 @@ class AuthProtocol(object):
             kwargs['headers'].update(additional_headers)
 
         if body:
-            kwargs['body'] = json.dumps(body)
+            kwargs['body'] = jsonutils.dumps(body)
 
         try:
             conn.request(method, path, **kwargs)
@@ -306,7 +307,7 @@ class AuthProtocol(object):
             conn.close()
 
         try:
-            data = json.loads(body)
+            data = jsonutils.loads(body)
         except ValueError:
             LOG.debug('Keystone did not return json-encoded body')
             data = {}
@@ -454,7 +455,7 @@ class AuthProtocol(object):
 
         try:
             catalog = token_info['access']['serviceCatalog']
-            rval['X-Service-Catalog'] = json.dumps(catalog)
+            rval['X-Service-Catalog'] = jsonutils.dumps(catalog)
         except KeyError:
             pass
 

keystone/middleware/core.py

@@ -14,12 +14,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import json
-
 from keystone.common import serializer
 from keystone.common import wsgi
 from keystone import config
 from keystone import exception
+from keystone.openstack.common import jsonutils
 
 
 CONF = config.CONF
@@ -110,7 +109,7 @@ class JsonBodyMiddleware(wsgi.Middleware):
 
         params_parsed = {}
         try:
-            params_parsed = json.loads(params_json)
+            params_parsed = jsonutils.loads(params_json)
         except ValueError:
             e = exception.ValidationError(attribute='valid JSON',
                                           target='request body')
@@ -138,7 +137,7 @@ class XmlBodyMiddleware(wsgi.Middleware):
         incoming_xml = 'application/xml' in str(request.content_type)
         if incoming_xml and request.body:
             request.content_type = 'application/json'
-            request.body = json.dumps(serializer.from_xml(request.body))
+            request.body = jsonutils.dumps(serializer.from_xml(request.body))
 
     def process_response(self, request, response):
         """Transform the response from JSON to XML."""
@@ -146,7 +145,8 @@ class XmlBodyMiddleware(wsgi.Middleware):
         if outgoing_xml and response.body:
             response.content_type = 'application/xml'
             try:
-                response.body = serializer.to_xml(json.loads(response.body))
+                body_obj = jsonutils.loads(response.body)
+                response.body = serializer.to_xml(body_obj)
             except:
                 raise exception.Error(message=response.body)
         return response

keystone/middleware/s3_token.py

@@ -34,10 +34,10 @@ This WSGI component:
 """
 
 import httplib
-import json
 
 import webob
 
+from keystone.openstack.common import jsonutils
 from swift.common import utils as swift_utils
 
 
@@ -167,7 +167,7 @@ class S3Token(object):
         creds = {'credentials': {'access': access,
                                  'token': token,
                                  'signature': signature}}
-        creds_json = json.dumps(creds)
+        creds_json = jsonutils.dumps(creds)
         self.logger.debug('Connecting to Keystone sending this JSON: %s' %
                           creds_json)
         # NOTE(vish): We could save a call to keystone by having
@@ -190,7 +190,7 @@ class S3Token(object):
                           resp.status, output))
 
         try:
-            identity_info = json.loads(output)
+            identity_info = jsonutils.loads(output)
             token_id = str(identity_info['access']['token']['id'])
             tenant = identity_info['access']['token']['tenant']
         except (ValueError, KeyError):

keystone/openstack/common/jsonutils.py

@@ -0,0 +1,140 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 Justin Santa Barbara
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+'''
+JSON related utilities.
+
+This module provides a few things:
+
+    1) A handy function for getting an object down to something that can be
+    JSON serialized.  See to_primitive().
+
+    2) Wrappers around loads() and dumps().  The dumps() wrapper will
+    automatically use to_primitive() for you if needed.
+
+    3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
+    is available.
+'''
+
+
+import datetime
+import inspect
+import itertools
+import json
+import xmlrpclib
+
+
+def to_primitive(value, convert_instances=False, level=0):
+    """Convert a complex object into primitives.
+
+    Handy for JSON serialization. We can optionally handle instances,
+    but since this is a recursive function, we could have cyclical
+    data structures.
+
+    To handle cyclical data structures we could track the actual objects
+    visited in a set, but not all objects are hashable. Instead we just
+    track the depth of the object inspections and don't go too deep.
+
+    Therefore, convert_instances=True is lossy ... be aware.
+
+    """
+    nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod,
+             inspect.isfunction, inspect.isgeneratorfunction,
+             inspect.isgenerator, inspect.istraceback, inspect.isframe,
+             inspect.iscode, inspect.isbuiltin, inspect.isroutine,
+             inspect.isabstract]
+    for test in nasty:
+        if test(value):
+            return unicode(value)
+
+    # value of itertools.count doesn't get caught by inspects
+    # above and results in infinite loop when list(value) is called.
+    if type(value) == itertools.count:
+        return unicode(value)
+
+    # FIXME(vish): Workaround for LP bug 852095. Without this workaround,
+    #              tests that raise an exception in a mocked method that
+    #              has a @wrap_exception with a notifier will fail. If
+    #              we up the dependency to 0.5.4 (when it is released) we
+    #              can remove this workaround.
+    if getattr(value, '__module__', None) == 'mox':
+        return 'mock'
+
+    if level > 3:
+        return '?'
+
+    # The try block may not be necessary after the class check above,
+    # but just in case ...
+    try:
+        # It's not clear why xmlrpclib created their own DateTime type, but
+        # for our purposes, make it a datetime type which is explicitly
+        # handled
+        if isinstance(value, xmlrpclib.DateTime):
+            value = datetime.datetime(*tuple(value.timetuple())[:6])
+
+        if isinstance(value, (list, tuple)):
+            o = []
+            for v in value:
+                o.append(to_primitive(v, convert_instances=convert_instances,
+                                      level=level))
+            return o
+        elif isinstance(value, dict):
+            o = {}
+            for k, v in value.iteritems():
+                o[k] = to_primitive(v, convert_instances=convert_instances,
+                                    level=level)
+            return o
+        elif isinstance(value, datetime.datetime):
+            return str(value)
+        elif hasattr(value, 'iteritems'):
+            return to_primitive(dict(value.iteritems()),
+                                convert_instances=convert_instances,
+                                level=level)
+        elif hasattr(value, '__iter__'):
+            return to_primitive(list(value), level)
+        elif convert_instances and hasattr(value, '__dict__'):
+            # Likely an instance of something. Watch for cycles.
+            # Ignore class member vars.
+            return to_primitive(value.__dict__,
+                                convert_instances=convert_instances,
+                                level=level + 1)
+        else:
+            return value
+    except TypeError, e:
+        # Class objects are tricky since they may define something like
+        # __iter__ defined but it isn't callable as list().
+        return unicode(value)
+
+
+def dumps(value, default=to_primitive, **kwargs):
+    return json.dumps(value, default=default, **kwargs)
+
+
+def loads(s):
+    return json.loads(s)
+
+
+try:
+    import anyjson
+except ImportError:
+    pass
+else:
+    anyjson._modules.append((__name__, 'dumps', TypeError,
+                                       'loads', ValueError))
+    anyjson.force_implementation(__name__)

openstack-common.conf

@@ -1,7 +1,7 @@
 [DEFAULT]
 
 # The list of modules to copy from openstack-common
-modules=cfg,importutils,iniparser,setup
+modules=cfg,importutils,iniparser,jsonutils,setup
 
 # The base module to hold the copy of openstack.common
 base=keystone

tests/test_auth_token_middleware.py

@@ -14,13 +14,12 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import json
-
 import webob
 import datetime
 import iso8601
 
 from keystone.middleware import auth_token
+from keystone.openstack.common import jsonutils
 from keystone import test
 
 
@@ -147,7 +146,7 @@ class FakeHTTPConnection(object):
         """
         if method == 'POST':
             status = 200
-            body = json.dumps({
+            body = jsonutils.dumps({
                 'access': {
                     'token': {'id': 'admin_token2'},
                 },
@@ -157,7 +156,7 @@ class FakeHTTPConnection(object):
             token_id = path.rsplit('/', 1)[1]
             if token_id in TOKEN_RESPONSES.keys():
                 status = 200
-                body = json.dumps(TOKEN_RESPONSES[token_id])
+                body = jsonutils.dumps(TOKEN_RESPONSES[token_id])
             else:
                 status = 404
                 body = str()

tests/test_content_types.py

@@ -15,14 +15,14 @@
 # under the License.
 
 import httplib
-import json
 import uuid
 
 from lxml import etree
 import nose.exc
 
-from keystone import test
 from keystone.common import serializer
+from keystone.openstack.common import jsonutils
+from keystone import test
 
 import default_fixtures
 
@@ -153,7 +153,7 @@ class RestfulTestCase(test.TestCase):
             headers['Accept'] = 'application/json'
             if body:
                 headers['Content-Type'] = 'application/json'
-                return json.dumps(body)
+                return jsonutils.dumps(body)
         elif content_type == 'xml':
             headers['Accept'] = 'application/xml'
             if body:
@@ -173,7 +173,7 @@ class RestfulTestCase(test.TestCase):
             self.assertIn(self.content_type, header)
 
             if self.content_type == 'json':
-                response.body = json.loads(response.body)
+                response.body = jsonutils.loads(response.body)
             elif self.content_type == 'xml':
                 response.body = etree.fromstring(response.body)
 

tests/test_exception.py

@@ -13,11 +13,12 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+
 import uuid
-import json
 
 from keystone.common import wsgi
 from keystone import exception
+from keystone.openstack.common import jsonutils
 from keystone import test
 
 
@@ -33,7 +34,7 @@ class ExceptionTestCase(test.TestCase):
         self.assertEqual(resp.status_int, e.code)
         self.assertEqual(resp.status, '%s %s' % (e.code, e.title))
 
-        j = json.loads(resp.body)
+        j = jsonutils.loads(resp.body)
         self.assertIsNotNone(j.get('error'))
         self.assertIsNotNone(j['error'].get('code'))
         self.assertIsNotNone(j['error'].get('title'))

tests/test_middleware.py

@@ -14,13 +14,12 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import json
-
 import webob
 
 from keystone import config
 from keystone import middleware
 from keystone import test
+from keystone.openstack.common import jsonutils
 
 
 CONF = config.CONF
@@ -151,7 +150,7 @@ class XmlBodyMiddlewareTest(test.TestCase):
                 method='POST')
         middleware.XmlBodyMiddleware(None).process_request(req)
         self.assertTrue(req.content_type, 'application/json')
-        self.assertTrue(json.loads(req.body))
+        self.assertTrue(jsonutils.loads(req.body))
 
     def test_json_unnaffected(self):
         """JSON-only requests should be unnaffected by the XML middleware."""

tests/test_s3_token_middleware.py

@@ -14,7 +14,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import json
 import logging
 
 import stubout
@@ -24,6 +23,7 @@ import webob
 from swift.common import utils as swift_utils
 
 from keystone.middleware import s3_token
+from keystone.openstack.common import jsonutils
 
 
 def denied_request(code):
@@ -74,7 +74,7 @@ class FakeHTTPConnection(object):
             raise Exception
         ret = {'access': {'token': {'id': 'TOKEN_ID',
                                     'tenant': {'id':  'TENANT_ID'}}}}
-        body = json.dumps(ret)
+        body = jsonutils.dumps(ret)
         status = self.status
         self.resp = FakeHTTPResponse(status, body)
 

tests/test_versions.py

@@ -15,10 +15,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
-
-from keystone import test
 from keystone import config
+from keystone import test
+from keystone.openstack.common import jsonutils
 
 
 CONF = config.CONF
@@ -38,7 +37,7 @@ class VersionTestCase(test.TestCase):
         client = self.client(self.public_app)
         resp = client.get('/')
         self.assertEqual(resp.status_int, 300)
-        data = json.loads(resp.body)
+        data = jsonutils.loads(resp.body)
         expected = {
             "versions": {
                 "values": [
@@ -86,7 +85,7 @@ class VersionTestCase(test.TestCase):
         client = self.client(self.admin_app)
         resp = client.get('/')
         self.assertEqual(resp.status_int, 300)
-        data = json.loads(resp.body)
+        data = jsonutils.loads(resp.body)
         expected = {
             "versions": {
                 "values": [

tests/test_wsgi.py

@@ -14,12 +14,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import json
-
 import webob
 
 from keystone import test
 from keystone.common import wsgi
+from keystone.openstack.common import jsonutils
 
 
 class ApplicationTest(test.TestCase):
@@ -47,4 +46,4 @@ class ApplicationTest(test.TestCase):
         app = FakeApp()
         req = self._make_request(url='/?1=2')
         resp = req.get_response(app)
-        self.assertEqual(json.loads(resp.body), {'1': '2'})
+        self.assertEqual(jsonutils.loads(resp.body), {'1': '2'})