Switch to oslo.serialization
Change-Id: I24be947cfc2d9ce60e65054e6562e544b1e21195
This commit is contained in:
parent
2b6e24f81b
commit
8d242bd7ab
openstack-common.confrequirements-py3.txtrequirements.txt
oslo/messaging
_drivers
notify
openstack/common
tests
@ -2,7 +2,6 @@
|
|||||||
|
|
||||||
# The list of modules to copy from oslo-incubator.git
|
# The list of modules to copy from oslo-incubator.git
|
||||||
module=gettextutils
|
module=gettextutils
|
||||||
module=jsonutils
|
|
||||||
module=middleware/base
|
module=middleware/base
|
||||||
module=middleware/__init__
|
module=middleware/__init__
|
||||||
module=context
|
module=context
|
||||||
|
@ -25,7 +25,7 @@ import six
|
|||||||
from oslo import messaging
|
from oslo import messaging
|
||||||
from oslo.messaging import _utils as utils
|
from oslo.messaging import _utils as utils
|
||||||
from oslo.messaging.openstack.common.gettextutils import _
|
from oslo.messaging.openstack.common.gettextutils import _
|
||||||
from oslo.messaging.openstack.common import jsonutils
|
from oslo.serialization import jsonutils
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -27,7 +27,7 @@ from oslo.messaging._drivers import amqpdriver
|
|||||||
from oslo.messaging._drivers import common as rpc_common
|
from oslo.messaging._drivers import common as rpc_common
|
||||||
from oslo.messaging import exceptions
|
from oslo.messaging import exceptions
|
||||||
from oslo.messaging.openstack.common.gettextutils import _
|
from oslo.messaging.openstack.common.gettextutils import _
|
||||||
from oslo.messaging.openstack.common import jsonutils
|
from oslo.serialization import jsonutils
|
||||||
from oslo.utils import importutils
|
from oslo.utils import importutils
|
||||||
from oslo.utils import netutils
|
from oslo.utils import netutils
|
||||||
|
|
||||||
|
@ -33,7 +33,7 @@ from oslo.messaging._drivers import base
|
|||||||
from oslo.messaging._drivers import common as rpc_common
|
from oslo.messaging._drivers import common as rpc_common
|
||||||
from oslo.messaging._executors import impl_eventlet # FIXME(markmc)
|
from oslo.messaging._executors import impl_eventlet # FIXME(markmc)
|
||||||
from oslo.messaging.openstack.common.gettextutils import _
|
from oslo.messaging.openstack.common.gettextutils import _
|
||||||
from oslo.messaging.openstack.common import jsonutils
|
from oslo.serialization import jsonutils
|
||||||
from oslo.utils import excutils
|
from oslo.utils import excutils
|
||||||
from oslo.utils import importutils
|
from oslo.utils import importutils
|
||||||
|
|
||||||
|
@ -31,8 +31,8 @@ from oslo import messaging
|
|||||||
from oslo.messaging._drivers import base
|
from oslo.messaging._drivers import base
|
||||||
from oslo.messaging._drivers import common
|
from oslo.messaging._drivers import common
|
||||||
from oslo.messaging._drivers.protocols.amqp import controller
|
from oslo.messaging._drivers.protocols.amqp import controller
|
||||||
from oslo.messaging.openstack.common import jsonutils
|
|
||||||
from oslo.messaging import target as messaging_target
|
from oslo.messaging import target as messaging_target
|
||||||
|
from oslo.serialization import jsonutils
|
||||||
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
@ -18,7 +18,7 @@
|
|||||||
import logging
|
import logging
|
||||||
|
|
||||||
from oslo.messaging.notify import notifier
|
from oslo.messaging.notify import notifier
|
||||||
from oslo.messaging.openstack.common import jsonutils
|
from oslo.serialization import jsonutils
|
||||||
|
|
||||||
|
|
||||||
class LogDriver(notifier._Driver):
|
class LogDriver(notifier._Driver):
|
||||||
|
@ -1,73 +0,0 @@
|
|||||||
# Copyright 2011 OpenStack Foundation.
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
"""
|
|
||||||
Import related utilities and helper functions.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import traceback
|
|
||||||
|
|
||||||
|
|
||||||
def import_class(import_str):
|
|
||||||
"""Returns a class from a string including module and class."""
|
|
||||||
mod_str, _sep, class_str = import_str.rpartition('.')
|
|
||||||
__import__(mod_str)
|
|
||||||
try:
|
|
||||||
return getattr(sys.modules[mod_str], class_str)
|
|
||||||
except AttributeError:
|
|
||||||
raise ImportError('Class %s cannot be found (%s)' %
|
|
||||||
(class_str,
|
|
||||||
traceback.format_exception(*sys.exc_info())))
|
|
||||||
|
|
||||||
|
|
||||||
def import_object(import_str, *args, **kwargs):
|
|
||||||
"""Import a class and return an instance of it."""
|
|
||||||
return import_class(import_str)(*args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def import_object_ns(name_space, import_str, *args, **kwargs):
|
|
||||||
"""Tries to import object from default namespace.
|
|
||||||
|
|
||||||
Imports a class and return an instance of it, first by trying
|
|
||||||
to find the class in a default namespace, then failing back to
|
|
||||||
a full path if not found in the default namespace.
|
|
||||||
"""
|
|
||||||
import_value = "%s.%s" % (name_space, import_str)
|
|
||||||
try:
|
|
||||||
return import_class(import_value)(*args, **kwargs)
|
|
||||||
except ImportError:
|
|
||||||
return import_class(import_str)(*args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def import_module(import_str):
|
|
||||||
"""Import a module."""
|
|
||||||
__import__(import_str)
|
|
||||||
return sys.modules[import_str]
|
|
||||||
|
|
||||||
|
|
||||||
def import_versioned_module(version, submodule=None):
|
|
||||||
module = 'oslo.messaging.v%s' % version
|
|
||||||
if submodule:
|
|
||||||
module = '.'.join((module, submodule))
|
|
||||||
return import_module(module)
|
|
||||||
|
|
||||||
|
|
||||||
def try_import(import_str, default=None):
|
|
||||||
"""Try to import a module and if it fails return default."""
|
|
||||||
try:
|
|
||||||
return import_module(import_str)
|
|
||||||
except ImportError:
|
|
||||||
return default
|
|
@ -1,196 +0,0 @@
|
|||||||
# Copyright 2010 United States Government as represented by the
|
|
||||||
# Administrator of the National Aeronautics and Space Administration.
|
|
||||||
# Copyright 2011 Justin Santa Barbara
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
'''
|
|
||||||
JSON related utilities.
|
|
||||||
|
|
||||||
This module provides a few things:
|
|
||||||
|
|
||||||
1) A handy function for getting an object down to something that can be
|
|
||||||
JSON serialized. See to_primitive().
|
|
||||||
|
|
||||||
2) Wrappers around loads() and dumps(). The dumps() wrapper will
|
|
||||||
automatically use to_primitive() for you if needed.
|
|
||||||
|
|
||||||
3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
|
|
||||||
is available.
|
|
||||||
'''
|
|
||||||
|
|
||||||
|
|
||||||
import codecs
|
|
||||||
import datetime
|
|
||||||
import functools
|
|
||||||
import inspect
|
|
||||||
import itertools
|
|
||||||
import sys
|
|
||||||
|
|
||||||
is_simplejson = False
|
|
||||||
if sys.version_info < (2, 7):
|
|
||||||
# On Python <= 2.6, json module is not C boosted, so try to use
|
|
||||||
# simplejson module if available
|
|
||||||
try:
|
|
||||||
import simplejson as json
|
|
||||||
is_simplejson = True
|
|
||||||
except ImportError:
|
|
||||||
import json
|
|
||||||
else:
|
|
||||||
import json
|
|
||||||
|
|
||||||
import six
|
|
||||||
import six.moves.xmlrpc_client as xmlrpclib
|
|
||||||
|
|
||||||
from oslo.messaging.openstack.common import gettextutils
|
|
||||||
from oslo.messaging.openstack.common import importutils
|
|
||||||
from oslo.messaging.openstack.common import strutils
|
|
||||||
from oslo.messaging.openstack.common import timeutils
|
|
||||||
|
|
||||||
netaddr = importutils.try_import("netaddr")
|
|
||||||
|
|
||||||
_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
|
|
||||||
inspect.isfunction, inspect.isgeneratorfunction,
|
|
||||||
inspect.isgenerator, inspect.istraceback, inspect.isframe,
|
|
||||||
inspect.iscode, inspect.isbuiltin, inspect.isroutine,
|
|
||||||
inspect.isabstract]
|
|
||||||
|
|
||||||
_simple_types = (six.string_types + six.integer_types
|
|
||||||
+ (type(None), bool, float))
|
|
||||||
|
|
||||||
|
|
||||||
def to_primitive(value, convert_instances=False, convert_datetime=True,
|
|
||||||
level=0, max_depth=3):
|
|
||||||
"""Convert a complex object into primitives.
|
|
||||||
|
|
||||||
Handy for JSON serialization. We can optionally handle instances,
|
|
||||||
but since this is a recursive function, we could have cyclical
|
|
||||||
data structures.
|
|
||||||
|
|
||||||
To handle cyclical data structures we could track the actual objects
|
|
||||||
visited in a set, but not all objects are hashable. Instead we just
|
|
||||||
track the depth of the object inspections and don't go too deep.
|
|
||||||
|
|
||||||
Therefore, convert_instances=True is lossy ... be aware.
|
|
||||||
|
|
||||||
"""
|
|
||||||
# handle obvious types first - order of basic types determined by running
|
|
||||||
# full tests on nova project, resulting in the following counts:
|
|
||||||
# 572754 <type 'NoneType'>
|
|
||||||
# 460353 <type 'int'>
|
|
||||||
# 379632 <type 'unicode'>
|
|
||||||
# 274610 <type 'str'>
|
|
||||||
# 199918 <type 'dict'>
|
|
||||||
# 114200 <type 'datetime.datetime'>
|
|
||||||
# 51817 <type 'bool'>
|
|
||||||
# 26164 <type 'list'>
|
|
||||||
# 6491 <type 'float'>
|
|
||||||
# 283 <type 'tuple'>
|
|
||||||
# 19 <type 'long'>
|
|
||||||
if isinstance(value, _simple_types):
|
|
||||||
return value
|
|
||||||
|
|
||||||
if isinstance(value, datetime.datetime):
|
|
||||||
if convert_datetime:
|
|
||||||
return timeutils.strtime(value)
|
|
||||||
else:
|
|
||||||
return value
|
|
||||||
|
|
||||||
# value of itertools.count doesn't get caught by nasty_type_tests
|
|
||||||
# and results in infinite loop when list(value) is called.
|
|
||||||
if type(value) == itertools.count:
|
|
||||||
return six.text_type(value)
|
|
||||||
|
|
||||||
# FIXME(vish): Workaround for LP bug 852095. Without this workaround,
|
|
||||||
# tests that raise an exception in a mocked method that
|
|
||||||
# has a @wrap_exception with a notifier will fail. If
|
|
||||||
# we up the dependency to 0.5.4 (when it is released) we
|
|
||||||
# can remove this workaround.
|
|
||||||
if getattr(value, '__module__', None) == 'mox':
|
|
||||||
return 'mock'
|
|
||||||
|
|
||||||
if level > max_depth:
|
|
||||||
return '?'
|
|
||||||
|
|
||||||
# The try block may not be necessary after the class check above,
|
|
||||||
# but just in case ...
|
|
||||||
try:
|
|
||||||
recursive = functools.partial(to_primitive,
|
|
||||||
convert_instances=convert_instances,
|
|
||||||
convert_datetime=convert_datetime,
|
|
||||||
level=level,
|
|
||||||
max_depth=max_depth)
|
|
||||||
if isinstance(value, dict):
|
|
||||||
return dict((k, recursive(v)) for k, v in six.iteritems(value))
|
|
||||||
elif isinstance(value, (list, tuple)):
|
|
||||||
return [recursive(lv) for lv in value]
|
|
||||||
|
|
||||||
# It's not clear why xmlrpclib created their own DateTime type, but
|
|
||||||
# for our purposes, make it a datetime type which is explicitly
|
|
||||||
# handled
|
|
||||||
if isinstance(value, xmlrpclib.DateTime):
|
|
||||||
value = datetime.datetime(*tuple(value.timetuple())[:6])
|
|
||||||
|
|
||||||
if convert_datetime and isinstance(value, datetime.datetime):
|
|
||||||
return timeutils.strtime(value)
|
|
||||||
elif isinstance(value, gettextutils.Message):
|
|
||||||
return value.data
|
|
||||||
elif hasattr(value, 'iteritems'):
|
|
||||||
return recursive(dict(value.iteritems()), level=level + 1)
|
|
||||||
elif hasattr(value, '__iter__'):
|
|
||||||
return recursive(list(value))
|
|
||||||
elif convert_instances and hasattr(value, '__dict__'):
|
|
||||||
# Likely an instance of something. Watch for cycles.
|
|
||||||
# Ignore class member vars.
|
|
||||||
return recursive(value.__dict__, level=level + 1)
|
|
||||||
elif netaddr and isinstance(value, netaddr.IPAddress):
|
|
||||||
return six.text_type(value)
|
|
||||||
else:
|
|
||||||
if any(test(value) for test in _nasty_type_tests):
|
|
||||||
return six.text_type(value)
|
|
||||||
return value
|
|
||||||
except TypeError:
|
|
||||||
# Class objects are tricky since they may define something like
|
|
||||||
# __iter__ defined but it isn't callable as list().
|
|
||||||
return six.text_type(value)
|
|
||||||
|
|
||||||
|
|
||||||
def dumps(value, default=to_primitive, **kwargs):
|
|
||||||
if is_simplejson:
|
|
||||||
kwargs['namedtuple_as_object'] = False
|
|
||||||
return json.dumps(value, default=default, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def dump(obj, fp, *args, **kwargs):
|
|
||||||
if is_simplejson:
|
|
||||||
kwargs['namedtuple_as_object'] = False
|
|
||||||
return json.dump(obj, fp, *args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def loads(s, encoding='utf-8', **kwargs):
|
|
||||||
return json.loads(strutils.safe_decode(s, encoding), **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def load(fp, encoding='utf-8', **kwargs):
|
|
||||||
return json.load(codecs.getreader(encoding)(fp), **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
try:
|
|
||||||
import anyjson
|
|
||||||
except ImportError:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
anyjson._modules.append((__name__, 'dumps', TypeError,
|
|
||||||
'loads', ValueError, 'load'))
|
|
||||||
anyjson.force_implementation(__name__)
|
|
@ -1,239 +0,0 @@
|
|||||||
# Copyright 2011 OpenStack Foundation.
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
"""
|
|
||||||
System-level utilities and helper functions.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import math
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
import unicodedata
|
|
||||||
|
|
||||||
import six
|
|
||||||
|
|
||||||
from oslo.messaging.openstack.common.gettextutils import _
|
|
||||||
|
|
||||||
|
|
||||||
UNIT_PREFIX_EXPONENT = {
|
|
||||||
'k': 1,
|
|
||||||
'K': 1,
|
|
||||||
'Ki': 1,
|
|
||||||
'M': 2,
|
|
||||||
'Mi': 2,
|
|
||||||
'G': 3,
|
|
||||||
'Gi': 3,
|
|
||||||
'T': 4,
|
|
||||||
'Ti': 4,
|
|
||||||
}
|
|
||||||
UNIT_SYSTEM_INFO = {
|
|
||||||
'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')),
|
|
||||||
'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')),
|
|
||||||
}
|
|
||||||
|
|
||||||
TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
|
|
||||||
FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
|
|
||||||
|
|
||||||
SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
|
|
||||||
SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
|
|
||||||
|
|
||||||
|
|
||||||
def int_from_bool_as_string(subject):
|
|
||||||
"""Interpret a string as a boolean and return either 1 or 0.
|
|
||||||
|
|
||||||
Any string value in:
|
|
||||||
|
|
||||||
('True', 'true', 'On', 'on', '1')
|
|
||||||
|
|
||||||
is interpreted as a boolean True.
|
|
||||||
|
|
||||||
Useful for JSON-decoded stuff and config file parsing
|
|
||||||
"""
|
|
||||||
return bool_from_string(subject) and 1 or 0
|
|
||||||
|
|
||||||
|
|
||||||
def bool_from_string(subject, strict=False, default=False):
|
|
||||||
"""Interpret a string as a boolean.
|
|
||||||
|
|
||||||
A case-insensitive match is performed such that strings matching 't',
|
|
||||||
'true', 'on', 'y', 'yes', or '1' are considered True and, when
|
|
||||||
`strict=False`, anything else returns the value specified by 'default'.
|
|
||||||
|
|
||||||
Useful for JSON-decoded stuff and config file parsing.
|
|
||||||
|
|
||||||
If `strict=True`, unrecognized values, including None, will raise a
|
|
||||||
ValueError which is useful when parsing values passed in from an API call.
|
|
||||||
Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
|
|
||||||
"""
|
|
||||||
if not isinstance(subject, six.string_types):
|
|
||||||
subject = six.text_type(subject)
|
|
||||||
|
|
||||||
lowered = subject.strip().lower()
|
|
||||||
|
|
||||||
if lowered in TRUE_STRINGS:
|
|
||||||
return True
|
|
||||||
elif lowered in FALSE_STRINGS:
|
|
||||||
return False
|
|
||||||
elif strict:
|
|
||||||
acceptable = ', '.join(
|
|
||||||
"'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
|
|
||||||
msg = _("Unrecognized value '%(val)s', acceptable values are:"
|
|
||||||
" %(acceptable)s") % {'val': subject,
|
|
||||||
'acceptable': acceptable}
|
|
||||||
raise ValueError(msg)
|
|
||||||
else:
|
|
||||||
return default
|
|
||||||
|
|
||||||
|
|
||||||
def safe_decode(text, incoming=None, errors='strict'):
|
|
||||||
"""Decodes incoming text/bytes string using `incoming` if they're not
|
|
||||||
already unicode.
|
|
||||||
|
|
||||||
:param incoming: Text's current encoding
|
|
||||||
:param errors: Errors handling policy. See here for valid
|
|
||||||
values http://docs.python.org/2/library/codecs.html
|
|
||||||
:returns: text or a unicode `incoming` encoded
|
|
||||||
representation of it.
|
|
||||||
:raises TypeError: If text is not an instance of str
|
|
||||||
"""
|
|
||||||
if not isinstance(text, (six.string_types, six.binary_type)):
|
|
||||||
raise TypeError("%s can't be decoded" % type(text))
|
|
||||||
|
|
||||||
if isinstance(text, six.text_type):
|
|
||||||
return text
|
|
||||||
|
|
||||||
if not incoming:
|
|
||||||
incoming = (sys.stdin.encoding or
|
|
||||||
sys.getdefaultencoding())
|
|
||||||
|
|
||||||
try:
|
|
||||||
return text.decode(incoming, errors)
|
|
||||||
except UnicodeDecodeError:
|
|
||||||
# Note(flaper87) If we get here, it means that
|
|
||||||
# sys.stdin.encoding / sys.getdefaultencoding
|
|
||||||
# didn't return a suitable encoding to decode
|
|
||||||
# text. This happens mostly when global LANG
|
|
||||||
# var is not set correctly and there's no
|
|
||||||
# default encoding. In this case, most likely
|
|
||||||
# python will use ASCII or ANSI encoders as
|
|
||||||
# default encodings but they won't be capable
|
|
||||||
# of decoding non-ASCII characters.
|
|
||||||
#
|
|
||||||
# Also, UTF-8 is being used since it's an ASCII
|
|
||||||
# extension.
|
|
||||||
return text.decode('utf-8', errors)
|
|
||||||
|
|
||||||
|
|
||||||
def safe_encode(text, incoming=None,
|
|
||||||
encoding='utf-8', errors='strict'):
|
|
||||||
"""Encodes incoming text/bytes string using `encoding`.
|
|
||||||
|
|
||||||
If incoming is not specified, text is expected to be encoded with
|
|
||||||
current python's default encoding. (`sys.getdefaultencoding`)
|
|
||||||
|
|
||||||
:param incoming: Text's current encoding
|
|
||||||
:param encoding: Expected encoding for text (Default UTF-8)
|
|
||||||
:param errors: Errors handling policy. See here for valid
|
|
||||||
values http://docs.python.org/2/library/codecs.html
|
|
||||||
:returns: text or a bytestring `encoding` encoded
|
|
||||||
representation of it.
|
|
||||||
:raises TypeError: If text is not an instance of str
|
|
||||||
"""
|
|
||||||
if not isinstance(text, (six.string_types, six.binary_type)):
|
|
||||||
raise TypeError("%s can't be encoded" % type(text))
|
|
||||||
|
|
||||||
if not incoming:
|
|
||||||
incoming = (sys.stdin.encoding or
|
|
||||||
sys.getdefaultencoding())
|
|
||||||
|
|
||||||
if isinstance(text, six.text_type):
|
|
||||||
return text.encode(encoding, errors)
|
|
||||||
elif text and encoding != incoming:
|
|
||||||
# Decode text before encoding it with `encoding`
|
|
||||||
text = safe_decode(text, incoming, errors)
|
|
||||||
return text.encode(encoding, errors)
|
|
||||||
else:
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def string_to_bytes(text, unit_system='IEC', return_int=False):
|
|
||||||
"""Converts a string into an float representation of bytes.
|
|
||||||
|
|
||||||
The units supported for IEC ::
|
|
||||||
|
|
||||||
Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it)
|
|
||||||
KB, KiB, MB, MiB, GB, GiB, TB, TiB
|
|
||||||
|
|
||||||
The units supported for SI ::
|
|
||||||
|
|
||||||
kb(it), Mb(it), Gb(it), Tb(it)
|
|
||||||
kB, MB, GB, TB
|
|
||||||
|
|
||||||
Note that the SI unit system does not support capital letter 'K'
|
|
||||||
|
|
||||||
:param text: String input for bytes size conversion.
|
|
||||||
:param unit_system: Unit system for byte size conversion.
|
|
||||||
:param return_int: If True, returns integer representation of text
|
|
||||||
in bytes. (default: decimal)
|
|
||||||
:returns: Numerical representation of text in bytes.
|
|
||||||
:raises ValueError: If text has an invalid value.
|
|
||||||
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
base, reg_ex = UNIT_SYSTEM_INFO[unit_system]
|
|
||||||
except KeyError:
|
|
||||||
msg = _('Invalid unit system: "%s"') % unit_system
|
|
||||||
raise ValueError(msg)
|
|
||||||
match = reg_ex.match(text)
|
|
||||||
if match:
|
|
||||||
magnitude = float(match.group(1))
|
|
||||||
unit_prefix = match.group(2)
|
|
||||||
if match.group(3) in ['b', 'bit']:
|
|
||||||
magnitude /= 8
|
|
||||||
else:
|
|
||||||
msg = _('Invalid string format: %s') % text
|
|
||||||
raise ValueError(msg)
|
|
||||||
if not unit_prefix:
|
|
||||||
res = magnitude
|
|
||||||
else:
|
|
||||||
res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix])
|
|
||||||
if return_int:
|
|
||||||
return int(math.ceil(res))
|
|
||||||
return res
|
|
||||||
|
|
||||||
|
|
||||||
def to_slug(value, incoming=None, errors="strict"):
|
|
||||||
"""Normalize string.
|
|
||||||
|
|
||||||
Convert to lowercase, remove non-word characters, and convert spaces
|
|
||||||
to hyphens.
|
|
||||||
|
|
||||||
Inspired by Django's `slugify` filter.
|
|
||||||
|
|
||||||
:param value: Text to slugify
|
|
||||||
:param incoming: Text's current encoding
|
|
||||||
:param errors: Errors handling policy. See here for valid
|
|
||||||
values http://docs.python.org/2/library/codecs.html
|
|
||||||
:returns: slugified unicode representation of `value`
|
|
||||||
:raises TypeError: If text is not an instance of str
|
|
||||||
"""
|
|
||||||
value = safe_decode(value, incoming, errors)
|
|
||||||
# NOTE(aababilov): no need to use safe_(encode|decode) here:
|
|
||||||
# encodings are always "ascii", error handling is always "ignore"
|
|
||||||
# and types are always known (first: unicode; second: str)
|
|
||||||
value = unicodedata.normalize("NFKD", value).encode(
|
|
||||||
"ascii", "ignore").decode("ascii")
|
|
||||||
value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
|
|
||||||
return SLUGIFY_HYPHENATE_RE.sub("-", value)
|
|
@ -1,210 +0,0 @@
|
|||||||
# Copyright 2011 OpenStack Foundation.
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
"""
|
|
||||||
Time related utilities and helper functions.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import calendar
|
|
||||||
import datetime
|
|
||||||
import time
|
|
||||||
|
|
||||||
import iso8601
|
|
||||||
import six
|
|
||||||
|
|
||||||
|
|
||||||
# ISO 8601 extended time format with microseconds
|
|
||||||
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
|
|
||||||
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
|
|
||||||
PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
|
|
||||||
|
|
||||||
|
|
||||||
def isotime(at=None, subsecond=False):
|
|
||||||
"""Stringify time in ISO 8601 format."""
|
|
||||||
if not at:
|
|
||||||
at = utcnow()
|
|
||||||
st = at.strftime(_ISO8601_TIME_FORMAT
|
|
||||||
if not subsecond
|
|
||||||
else _ISO8601_TIME_FORMAT_SUBSECOND)
|
|
||||||
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
|
|
||||||
st += ('Z' if tz == 'UTC' else tz)
|
|
||||||
return st
|
|
||||||
|
|
||||||
|
|
||||||
def parse_isotime(timestr):
|
|
||||||
"""Parse time from ISO 8601 format."""
|
|
||||||
try:
|
|
||||||
return iso8601.parse_date(timestr)
|
|
||||||
except iso8601.ParseError as e:
|
|
||||||
raise ValueError(six.text_type(e))
|
|
||||||
except TypeError as e:
|
|
||||||
raise ValueError(six.text_type(e))
|
|
||||||
|
|
||||||
|
|
||||||
def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
|
|
||||||
"""Returns formatted utcnow."""
|
|
||||||
if not at:
|
|
||||||
at = utcnow()
|
|
||||||
return at.strftime(fmt)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
|
|
||||||
"""Turn a formatted time back into a datetime."""
|
|
||||||
return datetime.datetime.strptime(timestr, fmt)
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_time(timestamp):
|
|
||||||
"""Normalize time in arbitrary timezone to UTC naive object."""
|
|
||||||
offset = timestamp.utcoffset()
|
|
||||||
if offset is None:
|
|
||||||
return timestamp
|
|
||||||
return timestamp.replace(tzinfo=None) - offset
|
|
||||||
|
|
||||||
|
|
||||||
def is_older_than(before, seconds):
|
|
||||||
"""Return True if before is older than seconds."""
|
|
||||||
if isinstance(before, six.string_types):
|
|
||||||
before = parse_strtime(before).replace(tzinfo=None)
|
|
||||||
else:
|
|
||||||
before = before.replace(tzinfo=None)
|
|
||||||
|
|
||||||
return utcnow() - before > datetime.timedelta(seconds=seconds)
|
|
||||||
|
|
||||||
|
|
||||||
def is_newer_than(after, seconds):
|
|
||||||
"""Return True if after is newer than seconds."""
|
|
||||||
if isinstance(after, six.string_types):
|
|
||||||
after = parse_strtime(after).replace(tzinfo=None)
|
|
||||||
else:
|
|
||||||
after = after.replace(tzinfo=None)
|
|
||||||
|
|
||||||
return after - utcnow() > datetime.timedelta(seconds=seconds)
|
|
||||||
|
|
||||||
|
|
||||||
def utcnow_ts():
|
|
||||||
"""Timestamp version of our utcnow function."""
|
|
||||||
if utcnow.override_time is None:
|
|
||||||
# NOTE(kgriffs): This is several times faster
|
|
||||||
# than going through calendar.timegm(...)
|
|
||||||
return int(time.time())
|
|
||||||
|
|
||||||
return calendar.timegm(utcnow().timetuple())
|
|
||||||
|
|
||||||
|
|
||||||
def utcnow():
|
|
||||||
"""Overridable version of utils.utcnow."""
|
|
||||||
if utcnow.override_time:
|
|
||||||
try:
|
|
||||||
return utcnow.override_time.pop(0)
|
|
||||||
except AttributeError:
|
|
||||||
return utcnow.override_time
|
|
||||||
return datetime.datetime.utcnow()
|
|
||||||
|
|
||||||
|
|
||||||
def iso8601_from_timestamp(timestamp):
|
|
||||||
"""Returns an iso8601 formatted date from timestamp."""
|
|
||||||
return isotime(datetime.datetime.utcfromtimestamp(timestamp))
|
|
||||||
|
|
||||||
|
|
||||||
utcnow.override_time = None
|
|
||||||
|
|
||||||
|
|
||||||
def set_time_override(override_time=None):
|
|
||||||
"""Overrides utils.utcnow.
|
|
||||||
|
|
||||||
Make it return a constant time or a list thereof, one at a time.
|
|
||||||
|
|
||||||
:param override_time: datetime instance or list thereof. If not
|
|
||||||
given, defaults to the current UTC time.
|
|
||||||
"""
|
|
||||||
utcnow.override_time = override_time or datetime.datetime.utcnow()
|
|
||||||
|
|
||||||
|
|
||||||
def advance_time_delta(timedelta):
|
|
||||||
"""Advance overridden time using a datetime.timedelta."""
|
|
||||||
assert utcnow.override_time is not None
|
|
||||||
try:
|
|
||||||
for dt in utcnow.override_time:
|
|
||||||
dt += timedelta
|
|
||||||
except TypeError:
|
|
||||||
utcnow.override_time += timedelta
|
|
||||||
|
|
||||||
|
|
||||||
def advance_time_seconds(seconds):
|
|
||||||
"""Advance overridden time by seconds."""
|
|
||||||
advance_time_delta(datetime.timedelta(0, seconds))
|
|
||||||
|
|
||||||
|
|
||||||
def clear_time_override():
|
|
||||||
"""Remove the overridden time."""
|
|
||||||
utcnow.override_time = None
|
|
||||||
|
|
||||||
|
|
||||||
def marshall_now(now=None):
|
|
||||||
"""Make an rpc-safe datetime with microseconds.
|
|
||||||
|
|
||||||
Note: tzinfo is stripped, but not required for relative times.
|
|
||||||
"""
|
|
||||||
if not now:
|
|
||||||
now = utcnow()
|
|
||||||
return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
|
|
||||||
minute=now.minute, second=now.second,
|
|
||||||
microsecond=now.microsecond)
|
|
||||||
|
|
||||||
|
|
||||||
def unmarshall_time(tyme):
|
|
||||||
"""Unmarshall a datetime dict."""
|
|
||||||
return datetime.datetime(day=tyme['day'],
|
|
||||||
month=tyme['month'],
|
|
||||||
year=tyme['year'],
|
|
||||||
hour=tyme['hour'],
|
|
||||||
minute=tyme['minute'],
|
|
||||||
second=tyme['second'],
|
|
||||||
microsecond=tyme['microsecond'])
|
|
||||||
|
|
||||||
|
|
||||||
def delta_seconds(before, after):
|
|
||||||
"""Return the difference between two timing objects.
|
|
||||||
|
|
||||||
Compute the difference in seconds between two date, time, or
|
|
||||||
datetime objects (as a float, to microsecond resolution).
|
|
||||||
"""
|
|
||||||
delta = after - before
|
|
||||||
return total_seconds(delta)
|
|
||||||
|
|
||||||
|
|
||||||
def total_seconds(delta):
|
|
||||||
"""Return the total seconds of datetime.timedelta object.
|
|
||||||
|
|
||||||
Compute total seconds of datetime.timedelta, datetime.timedelta
|
|
||||||
doesn't have method total_seconds in Python2.6, calculate it manually.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
return delta.total_seconds()
|
|
||||||
except AttributeError:
|
|
||||||
return ((delta.days * 24 * 3600) + delta.seconds +
|
|
||||||
float(delta.microseconds) / (10 ** 6))
|
|
||||||
|
|
||||||
|
|
||||||
def is_soon(dt, window):
|
|
||||||
"""Determines if time is going to happen in the next window seconds.
|
|
||||||
|
|
||||||
:param dt: the time
|
|
||||||
:param window: minimum seconds to remain to consider the time not soon
|
|
||||||
|
|
||||||
:return: True if expiration is within the given duration
|
|
||||||
"""
|
|
||||||
soon = (utcnow() + datetime.timedelta(seconds=window))
|
|
||||||
return normalize_time(dt) <= soon
|
|
@ -3,6 +3,7 @@
|
|||||||
# process, which may cause wedges in the gate later.
|
# process, which may cause wedges in the gate later.
|
||||||
|
|
||||||
oslo.config>=1.4.0 # Apache-2.0
|
oslo.config>=1.4.0 # Apache-2.0
|
||||||
|
oslo.serialization>=1.0.0 # Apache-2.0
|
||||||
oslo.utils>=1.0.0 # Apache-2.0
|
oslo.utils>=1.0.0 # Apache-2.0
|
||||||
stevedore>=1.0.0 # Apache-2.0
|
stevedore>=1.0.0 # Apache-2.0
|
||||||
|
|
||||||
|
@ -4,6 +4,7 @@
|
|||||||
|
|
||||||
oslo.config>=1.4.0 # Apache-2.0
|
oslo.config>=1.4.0 # Apache-2.0
|
||||||
oslo.utils>=1.0.0 # Apache-2.0
|
oslo.utils>=1.0.0 # Apache-2.0
|
||||||
|
oslo.serialization>=1.0.0 # Apache-2.0
|
||||||
stevedore>=1.0.0 # Apache-2.0
|
stevedore>=1.0.0 # Apache-2.0
|
||||||
|
|
||||||
# for jsonutils
|
# for jsonutils
|
||||||
|
@ -27,7 +27,7 @@ from oslo import messaging
|
|||||||
from oslo.messaging._drivers import amqpdriver
|
from oslo.messaging._drivers import amqpdriver
|
||||||
from oslo.messaging._drivers import common as driver_common
|
from oslo.messaging._drivers import common as driver_common
|
||||||
from oslo.messaging._drivers import impl_rabbit as rabbit_driver
|
from oslo.messaging._drivers import impl_rabbit as rabbit_driver
|
||||||
from oslo.messaging.openstack.common import jsonutils
|
from oslo.serialization import jsonutils
|
||||||
from tests import utils as test_utils
|
from tests import utils as test_utils
|
||||||
|
|
||||||
load_tests = testscenarios.load_tests_apply_scenarios
|
load_tests = testscenarios.load_tests_apply_scenarios
|
||||||
|
@ -30,8 +30,8 @@ from oslo.messaging.notify import _impl_log
|
|||||||
from oslo.messaging.notify import _impl_messaging
|
from oslo.messaging.notify import _impl_messaging
|
||||||
from oslo.messaging.notify import _impl_test
|
from oslo.messaging.notify import _impl_test
|
||||||
from oslo.messaging.notify import notifier as msg_notifier
|
from oslo.messaging.notify import notifier as msg_notifier
|
||||||
from oslo.messaging.openstack.common import jsonutils
|
|
||||||
from oslo.messaging import serializer as msg_serializer
|
from oslo.messaging import serializer as msg_serializer
|
||||||
|
from oslo.serialization import jsonutils
|
||||||
from oslo.utils import timeutils
|
from oslo.utils import timeutils
|
||||||
from tests import utils as test_utils
|
from tests import utils as test_utils
|
||||||
|
|
||||||
|
@ -20,7 +20,7 @@ import testscenarios
|
|||||||
|
|
||||||
from oslo import messaging
|
from oslo import messaging
|
||||||
from oslo.messaging._drivers import common as exceptions
|
from oslo.messaging._drivers import common as exceptions
|
||||||
from oslo.messaging.openstack.common import jsonutils
|
from oslo.serialization import jsonutils
|
||||||
from tests import utils as test_utils
|
from tests import utils as test_utils
|
||||||
|
|
||||||
load_tests = testscenarios.load_tests_apply_scenarios
|
load_tests = testscenarios.load_tests_apply_scenarios
|
||||||
|
Loading…
x
Reference in New Issue
Block a user