Use jsonutils from oslo incubator

Partial-Bug: #1261224
Change-Id: I9a1e4e661e11a423586869ca504c41028934f8de

heatclient/common/http.py

@@ -19,6 +19,7 @@ import os
 import posixpath
 import socket
 
+from heatclient.openstack.common import jsonutils
 from heatclient.openstack.common.py3kcompat import urlutils
 from six.moves import http_client as httplib
 
@@ -28,11 +29,6 @@ except ImportError:
     #TODO(bcwaldon): Handle this failure more gracefully
     pass
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
 
 from heatclient import exc
 
@@ -205,14 +201,14 @@ class HTTPClient(object):
         kwargs['headers'].setdefault('Accept', 'application/json')
 
         if 'body' in kwargs:
-            kwargs['body'] = json.dumps(kwargs['body'])
+            kwargs['body'] = jsonutils.dumps(kwargs['body'])
 
         resp, body_str = self._http_request(url, method, **kwargs)
 
         if 'application/json' in resp.getheader('content-type', None):
             body = body_str
             try:
-                body = json.loads(body)
+                body = jsonutils.loads(body)
             except ValueError:
                 LOG.error('Could not decode response body as JSON')
         else:

heatclient/common/utils.py

@@ -14,7 +14,6 @@
 #    under the License.
 from __future__ import print_function
 
-import json
 import os
 import prettytable
 import sys
@@ -24,9 +23,10 @@ import yaml
 
 from heatclient import exc
 from heatclient.openstack.common import importutils
+from heatclient.openstack.common import jsonutils
 
 supported_formats = {
-    "json": lambda x: json.dumps(x, indent=2),
+    "json": lambda x: jsonutils.dumps(x, indent=2),
     "yaml": yaml.safe_dump
 }
 
@@ -46,7 +46,7 @@ def link_formatter(links):
 
 
 def json_formatter(js):
-    return json.dumps(js, indent=2)
+    return jsonutils.dumps(js, indent=2)
 
 
 def text_wrap_formatter(d):

heatclient/exc.py

@@ -12,12 +12,9 @@
 
 import sys
 
-verbose = 0
+from heatclient.openstack.common import jsonutils
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
+verbose = 0
 
 
 class BaseException(Exception):
@@ -48,7 +45,7 @@ class HTTPException(BaseException):
     def __init__(self, message=None):
         super(HTTPException, self).__init__(message)
         try:
-            self.error = json.loads(message)
+            self.error = jsonutils.loads(message)
             if 'error' not in self.error:
                 raise KeyError('Key "error" not exists')
         except KeyError:

heatclient/openstack/common/importutils.py

@@ -1,6 +1,4 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright 2011 OpenStack LLC.
+# Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -24,7 +22,7 @@ import traceback
 
 
 def import_class(import_str):
-    """Returns a class from a string including module and class"""
+    """Returns a class from a string including module and class."""
     mod_str, _sep, class_str = import_str.rpartition('.')
     try:
         __import__(mod_str)
@@ -41,8 +39,9 @@ def import_object(import_str, *args, **kwargs):
 
 
 def import_object_ns(name_space, import_str, *args, **kwargs):
-    """
-    Import a class and return an instance of it, first by trying
+    """Tries to import object from default namespace.
+
+    Imports a class and return an instance of it, first by trying
     to find the class in a default namespace, then failing back to
     a full path if not found in the default namespace.
     """
@@ -57,3 +56,11 @@ def import_module(import_str):
     """Import a module."""
     __import__(import_str)
     return sys.modules[import_str]
+
+
+def try_import(import_str, default=None):
+    """Try to import a module and if it fails return default."""
+    try:
+        return import_module(import_str)
+    except ImportError:
+        return default

heatclient/openstack/common/jsonutils.py

@@ -0,0 +1,182 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 Justin Santa Barbara
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+'''
+JSON related utilities.
+
+This module provides a few things:
+
+    1) A handy function for getting an object down to something that can be
+    JSON serialized.  See to_primitive().
+
+    2) Wrappers around loads() and dumps().  The dumps() wrapper will
+    automatically use to_primitive() for you if needed.
+
+    3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
+    is available.
+'''
+
+
+import datetime
+import functools
+import inspect
+import itertools
+import json
+try:
+    import xmlrpclib
+except ImportError:
+    # NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3
+    #                 however the function and object call signatures
+    #                 remained the same. This whole try/except block should
+    #                 be removed and replaced with a call to six.moves once
+    #                 six 1.4.2 is released. See http://bit.ly/1bqrVzu
+    import xmlrpc.client as xmlrpclib
+
+import six
+
+from heatclient.openstack.common import gettextutils
+from heatclient.openstack.common import importutils
+from heatclient.openstack.common import timeutils
+
+netaddr = importutils.try_import("netaddr")
+
+_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
+                     inspect.isfunction, inspect.isgeneratorfunction,
+                     inspect.isgenerator, inspect.istraceback, inspect.isframe,
+                     inspect.iscode, inspect.isbuiltin, inspect.isroutine,
+                     inspect.isabstract]
+
+_simple_types = (six.string_types + six.integer_types
+                 + (type(None), bool, float))
+
+
+def to_primitive(value, convert_instances=False, convert_datetime=True,
+                 level=0, max_depth=3):
+    """Convert a complex object into primitives.
+
+    Handy for JSON serialization. We can optionally handle instances,
+    but since this is a recursive function, we could have cyclical
+    data structures.
+
+    To handle cyclical data structures we could track the actual objects
+    visited in a set, but not all objects are hashable. Instead we just
+    track the depth of the object inspections and don't go too deep.
+
+    Therefore, convert_instances=True is lossy ... be aware.
+
+    """
+    # handle obvious types first - order of basic types determined by running
+    # full tests on nova project, resulting in the following counts:
+    # 572754 <type 'NoneType'>
+    # 460353 <type 'int'>
+    # 379632 <type 'unicode'>
+    # 274610 <type 'str'>
+    # 199918 <type 'dict'>
+    # 114200 <type 'datetime.datetime'>
+    #  51817 <type 'bool'>
+    #  26164 <type 'list'>
+    #   6491 <type 'float'>
+    #    283 <type 'tuple'>
+    #     19 <type 'long'>
+    if isinstance(value, _simple_types):
+        return value
+
+    if isinstance(value, datetime.datetime):
+        if convert_datetime:
+            return timeutils.strtime(value)
+        else:
+            return value
+
+    # value of itertools.count doesn't get caught by nasty_type_tests
+    # and results in infinite loop when list(value) is called.
+    if type(value) == itertools.count:
+        return six.text_type(value)
+
+    # FIXME(vish): Workaround for LP bug 852095. Without this workaround,
+    #              tests that raise an exception in a mocked method that
+    #              has a @wrap_exception with a notifier will fail. If
+    #              we up the dependency to 0.5.4 (when it is released) we
+    #              can remove this workaround.
+    if getattr(value, '__module__', None) == 'mox':
+        return 'mock'
+
+    if level > max_depth:
+        return '?'
+
+    # The try block may not be necessary after the class check above,
+    # but just in case ...
+    try:
+        recursive = functools.partial(to_primitive,
+                                      convert_instances=convert_instances,
+                                      convert_datetime=convert_datetime,
+                                      level=level,
+                                      max_depth=max_depth)
+        if isinstance(value, dict):
+            return dict((k, recursive(v)) for k, v in six.iteritems(value))
+        elif isinstance(value, (list, tuple)):
+            return [recursive(lv) for lv in value]
+
+        # It's not clear why xmlrpclib created their own DateTime type, but
+        # for our purposes, make it a datetime type which is explicitly
+        # handled
+        if isinstance(value, xmlrpclib.DateTime):
+            value = datetime.datetime(*tuple(value.timetuple())[:6])
+
+        if convert_datetime and isinstance(value, datetime.datetime):
+            return timeutils.strtime(value)
+        elif isinstance(value, gettextutils.Message):
+            return value.data
+        elif hasattr(value, 'iteritems'):
+            return recursive(dict(value.iteritems()), level=level + 1)
+        elif hasattr(value, '__iter__'):
+            return recursive(list(value))
+        elif convert_instances and hasattr(value, '__dict__'):
+            # Likely an instance of something. Watch for cycles.
+            # Ignore class member vars.
+            return recursive(value.__dict__, level=level + 1)
+        elif netaddr and isinstance(value, netaddr.IPAddress):
+            return six.text_type(value)
+        else:
+            if any(test(value) for test in _nasty_type_tests):
+                return six.text_type(value)
+            return value
+    except TypeError:
+        # Class objects are tricky since they may define something like
+        # __iter__ defined but it isn't callable as list().
+        return six.text_type(value)
+
+
+def dumps(value, default=to_primitive, **kwargs):
+    return json.dumps(value, default=default, **kwargs)
+
+
+def loads(s):
+    return json.loads(s)
+
+
+def load(s):
+    return json.load(s)
+
+
+try:
+    import anyjson
+except ImportError:
+    pass
+else:
+    anyjson._modules.append((__name__, 'dumps', TypeError,
+                                       'loads', ValueError, 'load'))
+    anyjson.force_implementation(__name__)

heatclient/openstack/common/timeutils.py

@@ -0,0 +1,204 @@
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Time related utilities and helper functions.
+"""
+
+import calendar
+import datetime
+import time
+
+import iso8601
+import six
+
+
+# ISO 8601 extended time format with microseconds
+_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
+_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
+PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
+
+
+def isotime(at=None, subsecond=False):
+    """Stringify time in ISO 8601 format."""
+    if not at:
+        at = utcnow()
+    st = at.strftime(_ISO8601_TIME_FORMAT
+                     if not subsecond
+                     else _ISO8601_TIME_FORMAT_SUBSECOND)
+    tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
+    st += ('Z' if tz == 'UTC' else tz)
+    return st
+
+
+def parse_isotime(timestr):
+    """Parse time from ISO 8601 format."""
+    try:
+        return iso8601.parse_date(timestr)
+    except iso8601.ParseError as e:
+        raise ValueError(six.text_type(e))
+    except TypeError as e:
+        raise ValueError(six.text_type(e))
+
+
+def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
+    """Returns formatted utcnow."""
+    if not at:
+        at = utcnow()
+    return at.strftime(fmt)
+
+
+def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
+    """Turn a formatted time back into a datetime."""
+    return datetime.datetime.strptime(timestr, fmt)
+
+
+def normalize_time(timestamp):
+    """Normalize time in arbitrary timezone to UTC naive object."""
+    offset = timestamp.utcoffset()
+    if offset is None:
+        return timestamp
+    return timestamp.replace(tzinfo=None) - offset
+
+
+def is_older_than(before, seconds):
+    """Return True if before is older than seconds."""
+    if isinstance(before, six.string_types):
+        before = parse_strtime(before).replace(tzinfo=None)
+    return utcnow() - before > datetime.timedelta(seconds=seconds)
+
+
+def is_newer_than(after, seconds):
+    """Return True if after is newer than seconds."""
+    if isinstance(after, six.string_types):
+        after = parse_strtime(after).replace(tzinfo=None)
+    return after - utcnow() > datetime.timedelta(seconds=seconds)
+
+
+def utcnow_ts():
+    """Timestamp version of our utcnow function."""
+    if utcnow.override_time is None:
+        # NOTE(kgriffs): This is several times faster
+        # than going through calendar.timegm(...)
+        return int(time.time())
+
+    return calendar.timegm(utcnow().timetuple())
+
+
+def utcnow():
+    """Overridable version of utils.utcnow."""
+    if utcnow.override_time:
+        try:
+            return utcnow.override_time.pop(0)
+        except AttributeError:
+            return utcnow.override_time
+    return datetime.datetime.utcnow()
+
+
+def iso8601_from_timestamp(timestamp):
+    """Returns a iso8601 formated date from timestamp."""
+    return isotime(datetime.datetime.utcfromtimestamp(timestamp))
+
+
+utcnow.override_time = None
+
+
+def set_time_override(override_time=None):
+    """Overrides utils.utcnow.
+
+    Make it return a constant time or a list thereof, one at a time.
+
+    :param override_time: datetime instance or list thereof. If not
+                          given, defaults to the current UTC time.
+    """
+    utcnow.override_time = override_time or datetime.datetime.utcnow()
+
+
+def advance_time_delta(timedelta):
+    """Advance overridden time using a datetime.timedelta."""
+    assert(not utcnow.override_time is None)
+    try:
+        for dt in utcnow.override_time:
+            dt += timedelta
+    except TypeError:
+        utcnow.override_time += timedelta
+
+
+def advance_time_seconds(seconds):
+    """Advance overridden time by seconds."""
+    advance_time_delta(datetime.timedelta(0, seconds))
+
+
+def clear_time_override():
+    """Remove the overridden time."""
+    utcnow.override_time = None
+
+
+def marshall_now(now=None):
+    """Make an rpc-safe datetime with microseconds.
+
+    Note: tzinfo is stripped, but not required for relative times.
+    """
+    if not now:
+        now = utcnow()
+    return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
+                minute=now.minute, second=now.second,
+                microsecond=now.microsecond)
+
+
+def unmarshall_time(tyme):
+    """Unmarshall a datetime dict."""
+    return datetime.datetime(day=tyme['day'],
+                             month=tyme['month'],
+                             year=tyme['year'],
+                             hour=tyme['hour'],
+                             minute=tyme['minute'],
+                             second=tyme['second'],
+                             microsecond=tyme['microsecond'])
+
+
+def delta_seconds(before, after):
+    """Return the difference between two timing objects.
+
+    Compute the difference in seconds between two date, time, or
+    datetime objects (as a float, to microsecond resolution).
+    """
+    delta = after - before
+    return total_seconds(delta)
+
+
+def total_seconds(delta):
+    """Return the total seconds of datetime.timedelta object.
+
+    Compute total seconds of datetime.timedelta, datetime.timedelta
+    doesn't have method total_seconds in Python2.6, calculate it manually.
+    """
+    try:
+        return delta.total_seconds()
+    except AttributeError:
+        return ((delta.days * 24 * 3600) + delta.seconds +
+                float(delta.microseconds) / (10 ** 6))
+
+
+def is_soon(dt, window):
+    """Determines if time is going to happen in the next window seconds.
+
+    :params dt: the time
+    :params window: minimum seconds to remain to consider the time not soon
+
+    :return: True if expiration is within the given duration
+    """
+    soon = (utcnow() + datetime.timedelta(seconds=window))
+    return normalize_time(dt) <= soon

heatclient/tests/fakes.py

@@ -11,10 +11,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
-
 from heatclient.common import http
 from heatclient import exc
+from heatclient.openstack.common import jsonutils
 from keystoneclient.v2_0 import client as ksclient
 
 
@@ -49,7 +48,7 @@ def script_heat_list():
     resp = FakeHTTPResponse(200,
                             'success, you',
                             {'content-type': 'application/json'},
-                            json.dumps(resp_dict))
+                            jsonutils.dumps(resp_dict))
     http.HTTPClient.json_request('GET', '/stacks?').AndReturn(
         (resp, resp_dict))
 
@@ -68,9 +67,9 @@ def script_heat_normal_error():
     resp = FakeHTTPResponse(400,
                             'The resource could not be found',
                             {'content-type': 'application/json'},
-                            json.dumps(resp_dict))
+                            jsonutils.dumps(resp_dict))
     http.HTTPClient.json_request('GET', '/stacks/bad').AndRaise(
-        exc.from_response(resp, json.dumps(resp_dict)))
+        exc.from_response(resp, jsonutils.dumps(resp_dict)))
 
 
 def script_heat_error(resp_string):

heatclient/tests/test_shell.py

@@ -23,14 +23,11 @@ import tempfile
 import testscenarios
 import testtools
 
+from heatclient.openstack.common import jsonutils
 from heatclient.openstack.common.py3kcompat import urlutils
 from heatclient.openstack.common import strutils
 from mox3 import mox
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
 from keystoneclient.v2_0 import client as ksclient
 
 from heatclient.common import http
@@ -356,7 +353,7 @@ class ShellTestUserPass(ShellBase):
         }
 
         self._script_keystone_client()
-        fakes.script_heat_error(json.dumps(resp_dict))
+        fakes.script_heat_error(jsonutils.dumps(resp_dict))
 
         self.m.ReplayAll()
 
@@ -379,7 +376,7 @@ class ShellTestUserPass(ShellBase):
         }
 
         self._script_keystone_client()
-        fakes.script_heat_error(json.dumps(resp_dict))
+        fakes.script_heat_error(jsonutils.dumps(resp_dict))
 
         self.m.ReplayAll()
 
@@ -414,7 +411,7 @@ class ShellTestUserPass(ShellBase):
         }
 
         self._script_keystone_client()
-        fakes.script_heat_error(json.dumps(missing_message))
+        fakes.script_heat_error(jsonutils.dumps(missing_message))
         self.m.ReplayAll()
 
         try:
@@ -435,7 +432,7 @@ class ShellTestUserPass(ShellBase):
         }
 
         self._script_keystone_client()
-        fakes.script_heat_error(json.dumps(resp_dict))
+        fakes.script_heat_error(jsonutils.dumps(resp_dict))
         self.m.ReplayAll()
 
         exc.verbose = 1
@@ -458,7 +455,7 @@ class ShellTestUserPass(ShellBase):
             200,
             'OK',
             {'content-type': 'application/json'},
-            json.dumps(resp_dict))
+            jsonutils.dumps(resp_dict))
         http.HTTPClient.json_request(
             'GET', '/stacks/teststack/1').AndReturn((resp, resp_dict))
 
@@ -487,7 +484,7 @@ class ShellTestUserPass(ShellBase):
             'OK',
             {'content-type': 'application/json'},
             template_data)
-        resp_dict = json.loads(template_data)
+        resp_dict = jsonutils.loads(template_data)
         http.HTTPClient.json_request(
             'GET', '/stacks/teststack/template').AndReturn((resp, resp_dict))
 
@@ -515,7 +512,7 @@ class ShellTestUserPass(ShellBase):
             200,
             'OK',
             {'content-type': 'application/json'},
-            json.dumps(resp_dict))
+            jsonutils.dumps(resp_dict))
         http.HTTPClient.json_request(
             'GET', '/stacks/teststack/template').AndReturn((resp, resp_dict))
 
@@ -755,7 +752,7 @@ class ShellTestEvents(ShellBase):
             200,
             'OK',
             {'content-type': 'application/json'},
-            json.dumps(resp_dict))
+            jsonutils.dumps(resp_dict))
         stack_id = 'teststack/1'
         resource_name = 'testresource/1'
         http.HTTPClient.json_request(
@@ -811,7 +808,7 @@ class ShellTestEvents(ShellBase):
             200,
             'OK',
             {'content-type': 'application/json'},
-            json.dumps(resp_dict))
+            jsonutils.dumps(resp_dict))
         stack_id = 'teststack/1'
         resource_name = 'testresource/1'
         http.HTTPClient.json_request(

heatclient/v1/shell.py

@@ -13,12 +13,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
 import os
 import urllib
 import yaml
 
 from heatclient.common import utils
+from heatclient.openstack.common import jsonutils
 from heatclient.openstack.common.py3kcompat import urlutils
 
 import heatclient.exc as exc
@@ -28,7 +28,7 @@ def _set_template_fields(hc, args, fields):
     if args.template_file:
         tpl = open(args.template_file).read()
         if tpl.startswith('{'):
-            fields['template'] = json.loads(tpl)
+            fields['template'] = jsonutils.loads(tpl)
         else:
             fields['template'] = tpl
     elif args.template_url:
@@ -36,7 +36,7 @@ def _set_template_fields(hc, args, fields):
     elif args.template_object:
         template_body = hc.http_client.raw_request('GET', args.template_object)
         if template_body:
-            fields['template'] = json.loads(template_body)
+            fields['template'] = jsonutils.loads(template_body)
         else:
             raise exc.CommandError('Could not fetch template from %s'
                                    % args.template_object)
@@ -310,7 +310,7 @@ def do_resource_type_show(hc, args={}):
         raise exc.CommandError(
             'Resource Type not found: %s' % args.resource_type)
     else:
-        print(json.dumps(resource_type, indent=2))
+        print(jsonutils.dumps(resource_type, indent=2))
 
 
 @utils.arg('id', metavar='<NAME or ID>',
@@ -333,7 +333,7 @@ def do_template_show(hc, args):
         if 'heat_template_version' in template:
             print yaml.safe_dump(template, indent=2)
         else:
-            print json.dumps(template, indent=2)
+            print jsonutils.dumps(template, indent=2)
 
 
 @utils.arg('-u', '--template-url', metavar='<URL>',
@@ -374,7 +374,7 @@ def do_template_validate(hc, args):
     _process_environment_and_files(args, fields)
 
     validation = hc.stacks.validate(**fields)
-    print json.dumps(validation, indent=2)
+    print jsonutils.dumps(validation, indent=2)
 
 
 @utils.arg('id', metavar='<NAME or ID>',
@@ -459,7 +459,7 @@ def do_resource_metadata(hc, args):
         raise exc.CommandError('Stack or resource not found: %s %s' %
                                (args.id, args.resource))
     else:
-        print json.dumps(metadata, indent=2)
+        print jsonutils.dumps(metadata, indent=2)
 
 
 @utils.arg('id', metavar='<NAME or ID>',