630 lines
20 KiB
Python
630 lines
20 KiB
Python
import itertools
|
|
|
|
class _missing(object):
|
|
pass
|
|
|
|
class Invalid(Exception):
|
|
"""
|
|
An exception raised by data types and validators indicating that
|
|
the value for a particular structure was not valid.
|
|
|
|
The constructor receives a mandatory ``struct`` argument. This
|
|
must be an instance of the :class:`cereal.Structure` class.
|
|
|
|
The constructor also receives an optional ``msg`` keyword
|
|
argument, defaulting to ``None``. The ``msg`` argument is a
|
|
freeform field indicating the error circumstance.
|
|
"""
|
|
pos = None
|
|
parent = None
|
|
|
|
def __init__(self, struct, msg=None):
|
|
Exception.__init__(self, struct, msg)
|
|
self.struct = struct
|
|
self.msg = msg
|
|
self.children = []
|
|
|
|
def add(self, exc):
|
|
""" Add a child exception; ``exc`` must be an instance of
|
|
:class:`cereal.Invalid`"""
|
|
exc.parent = self
|
|
self.children.append(exc)
|
|
|
|
def paths(self):
|
|
""" Return all paths through the exception graph """
|
|
def traverse(node, stack):
|
|
stack.append(node)
|
|
|
|
if not node.children:
|
|
yield tuple(stack)
|
|
|
|
for child in node.children:
|
|
for path in traverse(child, stack):
|
|
yield path
|
|
|
|
stack.pop()
|
|
|
|
return traverse(self, [])
|
|
|
|
def _keyname(self):
|
|
if self.parent and isinstance(self.parent.struct.typ, Positional):
|
|
return str(self.pos)
|
|
return str(self.struct.name)
|
|
|
|
def asdict(self):
|
|
""" Return a dictionary containing an error report for this
|
|
exception"""
|
|
paths = self.paths()
|
|
errors = {}
|
|
for path in paths:
|
|
keyparts = []
|
|
msgs = []
|
|
for exc in path:
|
|
exc.msg and msgs.append(exc.msg)
|
|
keyname = exc._keyname()
|
|
keyname and keyparts.append(keyname)
|
|
errors['.'.join(keyparts)] = '; '.join(msgs)
|
|
return errors
|
|
|
|
class All(object):
|
|
""" Composite validator which succeeds if none of its
|
|
subvalidators raises an :class:`cereal.Invalid` exception"""
|
|
def __init__(self, *validators):
|
|
self.validators = validators
|
|
|
|
def __call__(self, struct, value):
|
|
msgs = []
|
|
for validator in self.validators:
|
|
try:
|
|
validator(struct, value)
|
|
except Invalid, e:
|
|
msgs.append(e.msg)
|
|
|
|
if msgs:
|
|
raise Invalid(struct, msgs)
|
|
|
|
class Range(object):
|
|
""" Validator which succeeds if the value it is passed is greater
|
|
or equal to ``min`` and less than or equal to ``max``. If ``min``
|
|
is not specified, or is specified as ``None``, no lower bound
|
|
exists. If ``max`` is not specified, or is specified as ``None``,
|
|
no upper bound exists."""
|
|
def __init__(self, min=None, max=None):
|
|
self.min = min
|
|
self.max = max
|
|
|
|
def __call__(self, struct, value):
|
|
if self.min is not None:
|
|
if value < self.min:
|
|
raise Invalid(
|
|
struct,
|
|
'%r is less than minimum value %r' % (value, self.min))
|
|
|
|
if self.max is not None:
|
|
if value > self.max:
|
|
raise Invalid(
|
|
struct,
|
|
'%r is greater than maximum value %r' % (value, self.max))
|
|
|
|
class OneOf(object):
|
|
""" Validator which succeeds if the value passed to it is one of
|
|
a fixed set of values """
|
|
def __init__(self, values):
|
|
self.values = values
|
|
|
|
def __call__(self, struct, value):
|
|
if not value in self.values:
|
|
raise Invalid(struct, '%r is not one of %r' % (value, self.values))
|
|
|
|
class Mapping(object):
|
|
""" A type which represents a mapping of names to structures.
|
|
|
|
The substructures of the :class:`cereal.Structure` that wraps this
|
|
type imply the named keys and values in the mapping.
|
|
|
|
The constructor of a mapping type accepts a single optional
|
|
keyword argument named ``unknown_keys``. By default, this
|
|
argument is ``ignore``.
|
|
|
|
The potential values of ``unknown_keys`` are:
|
|
|
|
- ``ignore`` means that keys that are not present in the schema
|
|
associated with this type will be ignored during
|
|
deserialization.
|
|
|
|
- ``raise`` will cause a :exc:`cereal.Invalid` exception to
|
|
be raised when unknown keys are present during deserialization.
|
|
|
|
- ``preserve`` will preserve the 'raw' unknown keys and values in
|
|
the returned data structure during deserialization.
|
|
"""
|
|
|
|
def __init__(self, unknown_keys='ignore'):
|
|
if not unknown_keys in ['ignore', 'raise', 'preserve']:
|
|
raise ValueError(
|
|
'unknown_keys argument must be one of "ignore", "raise", '
|
|
'or "preserve"')
|
|
self.unknown_keys = unknown_keys
|
|
|
|
def _validate(self, struct, value):
|
|
try:
|
|
return dict(value)
|
|
except Exception, e:
|
|
raise Invalid(struct, '%r is not a mapping type: %s' % (value, e))
|
|
|
|
def _impl(self, struct, value, callback, default_callback):
|
|
value = self._validate(struct, value)
|
|
|
|
error = None
|
|
result = {}
|
|
|
|
for num, substruct in enumerate(struct.structs):
|
|
name = substruct.name
|
|
subval = value.pop(name, _missing)
|
|
|
|
try:
|
|
if subval is _missing:
|
|
if substruct.required:
|
|
raise Invalid(
|
|
substruct,
|
|
'%r is required but missing' % substruct.name)
|
|
result[name] = default_callback(substruct)
|
|
else:
|
|
result[name] = callback(substruct, subval)
|
|
except Invalid, e:
|
|
if error is None:
|
|
error = Invalid(struct)
|
|
e.pos = num
|
|
error.add(e)
|
|
|
|
if self.unknown_keys == 'raise':
|
|
if value:
|
|
raise Invalid(struct,
|
|
'Unrecognized keys in mapping: %r' % value)
|
|
|
|
elif self.unknown_keys == 'preserve':
|
|
result.update(value)
|
|
|
|
if error is not None:
|
|
raise error
|
|
|
|
return result
|
|
|
|
def deserialize(self, struct, value):
|
|
def callback(substruct, subval):
|
|
return substruct.deserialize(subval)
|
|
def default_callback(substruct):
|
|
return substruct.default
|
|
return self._impl(struct, value, callback, default_callback)
|
|
|
|
def serialize(self, struct, value):
|
|
def callback(substruct, subval):
|
|
return substruct.serialize(subval)
|
|
def default_callback(substruct):
|
|
return substruct.serialize(substruct.default)
|
|
return self._impl(struct, value, callback, default_callback)
|
|
|
|
class Positional(object):
|
|
"""
|
|
Marker abstract base class meaning 'this type has children which
|
|
should be addressed by position instead of name' (e.g. via seq[0],
|
|
but never seq['name']). This is consulted by Invalid.asdict when
|
|
creating a dictionary representation of an error structure.
|
|
"""
|
|
|
|
class Tuple(Positional):
|
|
""" A type which represents a fixed-length sequence of structures.
|
|
|
|
The substructures of the :class:`cereal.Structure` that wraps this
|
|
type imply the positional elements of the tuple in the order they
|
|
are added.
|
|
|
|
This type is willing to serialize and deserialized iterables that,
|
|
when converted to a tuple, have the same number of elements as the
|
|
number of the associated structure's substructures.
|
|
"""
|
|
def _validate(self, struct, value):
|
|
if not hasattr(value, '__iter__'):
|
|
raise Invalid(struct, '%r is not iterable' % value)
|
|
|
|
valuelen, structlen = len(value), len(struct.structs)
|
|
|
|
if valuelen != structlen:
|
|
raise Invalid(
|
|
struct,
|
|
('%s has an incorrect number of elements '
|
|
'(expected %s, was %s)' % (value, structlen, valuelen)))
|
|
|
|
return list(value)
|
|
|
|
def _impl(self, struct, value, callback):
|
|
value = self._validate(struct, value)
|
|
error = None
|
|
result = []
|
|
|
|
for num, substruct in enumerate(struct.structs):
|
|
subval = value[num]
|
|
try:
|
|
result.append(callback(substruct, subval))
|
|
except Invalid, e:
|
|
if error is None:
|
|
error = Invalid(struct)
|
|
e.pos = num
|
|
error.add(e)
|
|
|
|
if error is not None:
|
|
raise error
|
|
|
|
return tuple(result)
|
|
|
|
def deserialize(self, struct, value):
|
|
def callback(substruct, subval):
|
|
return substruct.deserialize(subval)
|
|
return self._impl(struct, value, callback)
|
|
|
|
def serialize(self, struct, value):
|
|
def callback(substruct, subval):
|
|
return substruct.serialize(subval)
|
|
return self._impl(struct, value, callback)
|
|
|
|
class Sequence(Positional):
|
|
"""
|
|
A type which represents a variable-length sequence of structures,
|
|
all of which must be of the same type. This type is defined by the
|
|
the :class:`cereal.Structure` instance passed to the constructor
|
|
as ``struct``.
|
|
|
|
The ``struct`` argument to this type's constructor is required.
|
|
|
|
The substructures of the :class:`cereal.Structure` that wraps this
|
|
type are ignored.
|
|
"""
|
|
def __init__(self, struct):
|
|
self.struct = struct
|
|
|
|
def _validate(self, struct, value):
|
|
if not hasattr(value, '__iter__'):
|
|
raise Invalid(struct, '%r is not iterable' % value)
|
|
return list(value)
|
|
|
|
def _impl(self, struct, value, callback):
|
|
value = self._validate(struct, value)
|
|
|
|
error = None
|
|
result = []
|
|
for num, subval in enumerate(value):
|
|
try:
|
|
result.append(callback(self.struct, subval))
|
|
except Invalid, e:
|
|
if error is None:
|
|
error = Invalid(struct)
|
|
e.pos = num
|
|
error.add(e)
|
|
|
|
if error is not None:
|
|
raise error
|
|
|
|
return result
|
|
|
|
def deserialize(self, struct, value):
|
|
def callback(substruct, subval):
|
|
return substruct.deserialize(subval)
|
|
return self._impl(struct, value, callback)
|
|
|
|
def serialize(self, struct, value):
|
|
def callback(substruct, subval):
|
|
return substruct.serialize(subval)
|
|
return self._impl(struct, value, callback)
|
|
|
|
Seq = Sequence
|
|
|
|
default_encoding = 'utf-8'
|
|
|
|
class String(object):
|
|
""" A type representing a Unicode string. This type constructor
|
|
accepts a single argument ``encoding``, representing the encoding
|
|
which should be applied to object serialization. It defaults to
|
|
``utf-8`` if not provided.
|
|
|
|
The substructures of the :class:`cereal.Structure` that wraps this
|
|
type are ignored.
|
|
"""
|
|
def __init__(self, encoding=None):
|
|
self.encoding = encoding
|
|
|
|
def deserialize(self, struct, value):
|
|
try:
|
|
if isinstance(value, unicode):
|
|
return value
|
|
else:
|
|
return unicode(str(value), self.encoding or default_encoding)
|
|
except Exception, e:
|
|
raise Invalid(struct, '%r is not a string: %s' % (value, e))
|
|
|
|
def serialize(self, struct, value):
|
|
try:
|
|
return unicode(value).encode(self.encoding or default_encoding)
|
|
except Exception, e:
|
|
raise Invalid(struct,
|
|
'%r is cannot be serialized to str: %s' % (value, e))
|
|
|
|
Str = String
|
|
|
|
class Integer(object):
|
|
""" A type representing an integer.
|
|
|
|
The substructures of the :class:`cereal.Structure` that wraps this
|
|
type are ignored.
|
|
"""
|
|
def deserialize(self, struct, value):
|
|
try:
|
|
return int(value)
|
|
except Exception:
|
|
raise Invalid(struct, '%r is not a number' % value)
|
|
|
|
def serialize(self, struct, value):
|
|
try:
|
|
return str(int(value))
|
|
except Exception:
|
|
raise Invalid(struct, '%r is not a number' % value)
|
|
|
|
Int = Integer
|
|
|
|
class Float(object):
|
|
""" A type representing a float.
|
|
|
|
The substructures of the :class:`cereal.Structure` that wraps this
|
|
type are ignored.
|
|
"""
|
|
def deserialize(self, struct, value):
|
|
try:
|
|
return float(value)
|
|
except Exception:
|
|
raise Invalid(struct, '%r is not a number' % value)
|
|
|
|
def serialize(self, struct, value):
|
|
try:
|
|
return str(float(value))
|
|
except Exception:
|
|
raise Invalid(struct, '%r is not a number' % value)
|
|
|
|
Int = Integer
|
|
|
|
class Boolean(object):
|
|
""" A type representing a boolean object.
|
|
|
|
During deserialization, a value in the set (``false``, ``0``) will
|
|
be considered ``False``. Anything else is considered
|
|
``True``. Case is ignored.
|
|
|
|
Serialization will produce ``true`` or ``false`` based on the
|
|
value.
|
|
|
|
The substructures of the :class:`cereal.Structure` that wraps this
|
|
type are ignored.
|
|
"""
|
|
|
|
def deserialize(self, struct, value):
|
|
try:
|
|
value = str(value)
|
|
except:
|
|
raise Invalid(struct, '%r is not a string' % value)
|
|
value = value.lower()
|
|
if value in ('false', '0'):
|
|
return False
|
|
return True
|
|
|
|
def serialize(self, struct, value):
|
|
return value and 'true' or 'false'
|
|
|
|
Bool = Boolean
|
|
|
|
class GlobalObject(object):
|
|
""" A type representing an importable Python object. This type
|
|
serializes 'global' Python objects (objects which can be imported)
|
|
to dotted Python names.
|
|
|
|
Two dotted name styles are supported during deserialization:
|
|
|
|
- ``pkg_resources``-style dotted names where non-module attributes
|
|
of a module are separated from the rest of the path using a ':'
|
|
e.g. ``package.module:attr``.
|
|
|
|
- ``zope.dottedname``-style dotted names where non-module
|
|
attributes of a module are separated from the rest of the path
|
|
using a '.' e.g. ``package.module.attr``.
|
|
|
|
These styles can be used interchangeably. If the serialization
|
|
contains a ``:`` (colon), the ``pkg_resources`` resolution
|
|
mechanism will be chosen, otherwise the ``zope.dottedname``
|
|
resolution mechanism will be chosen.
|
|
|
|
The constructor accepts a single argument named ``package`` which
|
|
should be a Python module or package object; it is used when
|
|
*relative* dotted names are supplied to the ``deserialize``
|
|
method. A serialization which has a ``.`` (dot) or ``:`` (colon)
|
|
as its first character is treated as relative. E.g. if
|
|
``.minidom`` is supplied to ``deserialize``, and the ``package``
|
|
argument to this type was passed the ``xml`` module object, the
|
|
resulting import would be for ``xml.minidom``. If a relative
|
|
package name is supplied to ``deserialize``, and no ``package``
|
|
was supplied to the constructor, an :exc:`cereal.Invalid` error
|
|
will be raised.
|
|
|
|
The substructures of the :class:`cereal.Structure` that wraps this
|
|
type are ignored.
|
|
"""
|
|
def __init__(self, package):
|
|
self.package = package
|
|
|
|
def _pkg_resources_style(self, struct, value):
|
|
""" package.module:attr style """
|
|
import pkg_resources
|
|
if value.startswith('.') or value.startswith(':'):
|
|
if not self.package:
|
|
raise ImportError(
|
|
'relative name %r irresolveable without package' % value)
|
|
if value in ['.', ':']:
|
|
value = self.package.__name__
|
|
else:
|
|
value = self.package.__name__ + value
|
|
return pkg_resources.EntryPoint.parse(
|
|
'x=%s' % value).load(False)
|
|
|
|
def _zope_dottedname_style(self, struct, value):
|
|
""" package.module.attr style """
|
|
module = self.package and self.package.__name__ or None
|
|
if value == '.':
|
|
if self.package is None:
|
|
raise Invalid(
|
|
struct,
|
|
"relative name %r irresolveable without package" % value)
|
|
name = module.split('.')
|
|
else:
|
|
name = value.split('.')
|
|
if not name[0]:
|
|
if module is None:
|
|
raise Invalid(
|
|
struct,
|
|
"relative name %r irresolveable without package" %
|
|
value)
|
|
module = module.split('.')
|
|
name.pop(0)
|
|
while not name[0]:
|
|
module.pop()
|
|
name.pop(0)
|
|
name = module + name
|
|
|
|
used = name.pop(0)
|
|
found = __import__(used)
|
|
for n in name:
|
|
used += '.' + n
|
|
try:
|
|
found = getattr(found, n)
|
|
except AttributeError:
|
|
__import__(used)
|
|
found = getattr(found, n)
|
|
|
|
return found
|
|
|
|
def deserialize(self, struct, value):
|
|
if not isinstance(value, basestring):
|
|
raise Invalid(struct, '%r is not a string' % value)
|
|
try:
|
|
if ':' in value:
|
|
return self._pkg_resources_style(struct, value)
|
|
else:
|
|
return self._zope_dottedname_style(struct, value)
|
|
except ImportError:
|
|
raise Invalid(struct,
|
|
'The dotted name %r cannot be imported' % value)
|
|
|
|
def serialize(self, struct, value):
|
|
try:
|
|
return value.__name__
|
|
except AttributeError:
|
|
raise Invalid(struct, '%r has no __name__' % value)
|
|
|
|
class Structure(object):
|
|
"""
|
|
Fundamental building block of schemas.
|
|
|
|
The constructor accepts these arguments:
|
|
|
|
- ``typ`` (required): The 'type' for this structure. It should be
|
|
an instance of a class that implements the
|
|
:class:`cereal.interfaces.Type` interface.
|
|
|
|
- ``structs``: a sequence of substructures. If the substructures
|
|
of this structure are not known at construction time, they can
|
|
later be added via the ``add`` method.
|
|
|
|
- ``name``: The name of this structure.
|
|
|
|
- ``default``: The default value for this structure; if it is not
|
|
provided, this structure has no default value and it will be
|
|
considered 'required' (the ``required`` attribute will be True).
|
|
|
|
- ``validator``: Optional validator for this structure. It should be
|
|
an object that implements the
|
|
:class:`cereal.interfaces.Validator` interface.
|
|
"""
|
|
|
|
_counter = itertools.count()
|
|
|
|
def __new__(cls, *arg, **kw):
|
|
inst = object.__new__(cls)
|
|
inst._order = cls._counter.next()
|
|
return inst
|
|
|
|
def __init__(self, typ, *structs, **kw):
|
|
self.typ = typ
|
|
self.validator = kw.get('validator', None)
|
|
self.default = kw.get('default', _missing)
|
|
self.name = kw.get('name', '')
|
|
self.structs = list(structs)
|
|
|
|
@property
|
|
def required(self):
|
|
""" Property which returns true if this structure is required in the
|
|
schema """
|
|
return self.default is _missing
|
|
|
|
def deserialize(self, value):
|
|
""" Derialize the value based on the schema represented by this
|
|
structure """
|
|
value = self.typ.deserialize(self, value)
|
|
if self.validator is not None:
|
|
self.validator(self, value)
|
|
return value
|
|
|
|
def serialize(self, value):
|
|
""" Serialize the value based on the schema represented by this
|
|
structure """
|
|
return self.typ.serialize(self, value)
|
|
|
|
def add(self, struct):
|
|
""" Add a substructure to this structure """
|
|
self.structs.append(struct)
|
|
|
|
Struct = Structure
|
|
|
|
class _SchemaMeta(type):
|
|
def __init__(cls, name, bases, clsattrs):
|
|
structs = []
|
|
for name, value in clsattrs.items():
|
|
if isinstance(value, Structure):
|
|
value.name = name
|
|
structs.append((value._order, value))
|
|
cls.__schema_structures__ = structs
|
|
# Combine all attrs from this class and its subclasses.
|
|
extended = []
|
|
for c in cls.__mro__:
|
|
extended.extend(getattr(c, '__schema_structures__', []))
|
|
# Sort the attrs to maintain the order as defined, and assign to the
|
|
# class.
|
|
extended.sort()
|
|
cls.structs = [x[1] for x in extended]
|
|
|
|
class Schema(object):
|
|
struct_type = Mapping
|
|
__metaclass__ = _SchemaMeta
|
|
|
|
def __new__(cls, *args, **kw):
|
|
inst = object.__new__(Structure)
|
|
inst.name = None
|
|
inst._order = Structure._counter.next()
|
|
struct = cls.struct_type(*args, **kw)
|
|
inst.__init__(struct)
|
|
for s in cls.structs:
|
|
inst.add(s)
|
|
return inst
|
|
|
|
MappingSchema = Schema
|
|
|
|
class SequenceSchema(Schema):
|
|
struct_type = Sequence
|
|
|
|
class TupleSchema(Schema):
|
|
struct_type = Tuple
|