From f8dee4f914d91e38e57d770b5c5c5c2a41552ee2 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 9 Jun 2010 14:22:40 +0000 Subject: [PATCH] simplify defaulting --- CHANGES.txt | 80 ++++++-- colander/__init__.py | 212 ++++++++------------- colander/interfaces.py | 51 +++--- colander/tests.py | 155 ++-------------- docs/api.rst | 1 + docs/basics.rst | 26 +-- docs/defaults.rst | 405 ----------------------------------------- docs/extending.rst | 52 ++++-- docs/glossary.rst | 33 +++- docs/index.rst | 2 +- docs/null.rst | 306 +++++++++++++++++++++++++++++++ 11 files changed, 555 insertions(+), 768 deletions(-) delete mode 100644 docs/defaults.rst create mode 100644 docs/null.rst diff --git a/CHANGES.txt b/CHANGES.txt index a3601b8..a37a274 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -17,24 +17,68 @@ Next release - Raise a ``TypeError`` when bogus keyword arguments are passed to ``colander.SchemaNode``. -- Upgrade explanations required: ``partial`` argument and attribute of - colander.MappingSchema has been removed, ``null`` added to - serialization data structure for partials instead of omitting them - from output like before, ``missing`` constructor arg to SchemaNode, - nulls may be present in serialized and deserialized data structures, - ``sdefault`` attribute of SchemaNode has been removed, ``srequired`` - attribute of SchemaNode has been added, the ``value`` argument to - ``deserialize`` is now named ``cstruct``, the ``value`` argument`` - to ``serialize`` is now named ``appstruct``, types must now expect - ``colander.default`` instead of ``None`` during ``serialize`` as - ``appstruct``, types must now expect ``colander.default`` instead of - ``None`` during ``deserialize`` as ``cstruct``, - ``colander.SchemaNode.serialize`` and - ``colander.SchemaNode.deserialize`` now require no explicit value - argument (value defaults to ``colander.default``), ``allow_empty`` - argument of ``colander.String`` type removed (use ``missing=''`` - instead in the surrounding schemanode), serialization and - deserialization of ``null`` in ``colander.String`` returns null now. +- ``missing`` constructor arg to SchemaNode: signifies + *deserialization* default, disambiguated from ``default`` which acted + as both serialization and deserialization default previously. + + Changes necessitated / made possible by SchemaNode ``missing`` + addition: + + - The ``allow_empty`` argument of the ``colander.String`` type was + removed (use ``missing=''`` as a wrapper SchemaNode argument + instead). + +- New concept: ``colander.null`` input to serialization and + deserialization. Use of ``colander.null` normalizes serialization + and deserialization default handling. + + Changes necessitated / made possible by ``colander.null`` addition: + + - ``partial`` argument and attribute of colander.MappingSchema has + been removed; all serializations are partial, and partial + deserializations are not necessary. + + - ``colander.null`` values are added to the cstruct for partial + serializations instead of omitting missing node values from + the cstruct. + + - ``colander.null`` may now be present in serialized and + deserialized data structures. + + - ``sdefault`` attribute of SchemaNode has been removed; we never need + to serialize a default anymore. + + - The value ``colander.null`` will be passed as ``appstruct`` to + each type's ``serialize`` method when a mapping appstruct doesn't + have a corresponding key instead of ``None``, as was the practice + previously. + + - The value ``colander.null`` will be passed as ``cstruct`` to + each type's ``deserialize`` method when a mapping cstruct + doesn't have a corresponding key instead of ``None``, as was the + practice previously. + + - Types now must handle ``colander.null`` explicitly during + serialization. + +- Updated and expanded documentation, particularly with respect to new + ``colander.null`` handling. + +- The ``value`` argument`` to the ``serialize`` method of a SchemaNode + is now named ``appstruct``. It is no longer a required argument; it + defaults to ``colander.null`` now. + + The ``value`` argument to the ``deserialize`` method of a SchemaNode + is now named ``cstruct``. It is no longer a required argument; it + defaults to ``colander.null`` now. + +- The ``value`` argument to the ``serialize`` method of each built-in + type is now named ``appstruct``, and is now required: it is no + longer a keyword argument that has a default. + + The ``value`` argument to the ``deserialize`` method of each + built-in type is now named ``cstruct``, and is now required: it is + no longer a keyword argument that has a default. 0.6.2 (2010-05-08) ------------------ diff --git a/colander/__init__.py b/colander/__init__.py index 9a0a8ea..8ac6ca3 100644 --- a/colander/__init__.py +++ b/colander/__init__.py @@ -8,18 +8,6 @@ import translationstring _ = translationstring.TranslationStringFactory('colander') -class _marker(object): - def __repr__(self): - return '' - -_marker = _marker() - -class default(object): - def __repr__(self): - return '' - -default = default() - class null(object): def __nonzero__(self): return False @@ -361,16 +349,18 @@ class Mapping(object): Special behavior is exhibited when a subvalue of a mapping is present in the schema but is missing from the mapping passed to either the ``serialize`` or ``deserialize`` method of this class. - In this case, the :attr:`colander.default` value will be passed to - the schema node representing the subvalue of the mapping. During + In this case, the :attr:`colander.null` value will be passed to + the ``serialize`` or ``deserialize`` method of the schema node + representing the subvalue of the mapping respectively. During serialization, this will result in the behavior described in - :ref:`serializing_default` for the subnode. During - deserialization, this will result in the behavior described in - :ref:`deserializing_default` for the subnode. + :ref:`serializing_null` for the subnode. During deserialization, + this will result in the behavior described in + :ref:`deserializing_null` for the subnode. - If the :attr:`colander.null` value is passed to the serialize or - deserialize methods of this class, the empty dictionary will be - returned. + If the :attr:`colander.null` value is passed to the serialize + method of this class, a dictionary will be returned, where each of + the values in the returned dictionary is the serialized + representation of the null value for its type. """ def __init__(self, unknown='ignore'): self.unknown = unknown @@ -404,7 +394,7 @@ class Mapping(object): for num, subnode in enumerate(node.children): name = subnode.name - subval = value.pop(name, default) + subval = value.pop(name, null) try: result[name] = callback(subnode, subval) except Invalid, e: @@ -438,8 +428,6 @@ class Mapping(object): return self._impl(node, appstruct, callback) def deserialize(self, node, cstruct): - if cstruct is null: - return null def callback(subnode, subcstruct): return subnode.deserialize(subcstruct) @@ -465,9 +453,9 @@ class Tuple(Positional): when converted to a tuple, have the same number of elements as the number of the associated node's subnodes. - If the :attr:`colander.null` value is passed to the serialize or - deserialize methods of this class, the :attr:`colander.null` value - will be returned. + If the :attr:`colander.null` value is passed to the serialize + method of this class, the :attr:`colander.null` value will be + returned. """ def _validate(self, node, value): if not hasattr(value, '__iter__'): @@ -517,9 +505,6 @@ class Tuple(Positional): return self._impl(node, appstruct, callback) def deserialize(self, node, cstruct): - if cstruct is null: - return null - def callback(subnode, subval): return subnode.deserialize(subval) @@ -550,9 +535,8 @@ class Sequence(Positional): The default value of ``accept_scalar`` is ``False``. - If the :attr:`colander.null` value is passed to the serialize or - deserialize methods of this class, the :attr:`colander.null` value - is returned. + If the :attr:`colander.null` value is passed to the serialize + method of this class, the :attr:`colander.null` value is returned. """ def __init__(self, accept_scalar=False): self.accept_scalar = accept_scalar @@ -618,7 +602,6 @@ class Sequence(Positional): return self._impl(node, appstruct, callback, accept_scalar) def deserialize(self, node, cstruct, accept_scalar=None): - """ Along with the normal ``node`` and ``cstruct`` arguments, this method accepts an additional optional keyword argument: @@ -638,9 +621,6 @@ class Sequence(Positional): respect the default ``accept_scalar`` value attached to this instance via its constructor. """ - if cstruct is null: - return null - def callback(subnode, subcstruct): return subnode.deserialize(subcstruct) @@ -733,9 +713,6 @@ class String(object): mapping={'val':appstruct, 'err':e}) ) def deserialize(self, node, cstruct): - if cstruct is null: - return null - try: result = cstruct if not isinstance(result, unicode): @@ -770,10 +747,7 @@ class Number(object): mapping={'val':appstruct}), ) def deserialize(self, node, cstruct): - if cstruct is null: - return null - - if cstruct == '': + if not cstruct: raise Invalid(node, _('Required')) try: @@ -788,9 +762,9 @@ class Number(object): class Integer(Number): """ A type representing an integer. - If the :attr:`colander.null` value is passed to the serialize or - deserialize methods of this class, the :attr:`colander.null` value - will be returned. + If the :attr:`colander.null` value is passed to the serialize + method of this class, the :attr:`colander.null` value will be + returned. The subnodes of the :class:`colander.SchemaNode` that wraps this type are ignored. @@ -802,9 +776,9 @@ Int = Integer class Float(Number): """ A type representing a float. - If the :attr:`colander.null` value is passed to the serialize or - deserialize methods of this class, the :attr:`colander.null` value - will be returned. + If the :attr:`colander.null` value is passed to the serialize + method of this class, the :attr:`colander.null` value will be + returned. The subnodes of the :class:`colander.SchemaNode` that wraps this type are ignored. @@ -815,9 +789,9 @@ class Decimal(Number): """ A type representing a decimal floating point. Deserialization returns an instance of the Python ``decimal.Decimal`` type. - If the :attr:`colander.null` value is passed to the serialize or - deserialize methods of this class, the :attr:`colander.null` value - will be returned. + If the :attr:`colander.null` value is passed to the serialize + method of this class, the :attr:`colander.null` value will be + returned. The subnodes of the :class:`colander.SchemaNode` that wraps this type are ignored. @@ -835,9 +809,9 @@ class Boolean(object): Serialization will produce ``true`` or ``false`` based on the value. - If the :attr:`colander.null` value is passed to the serialize or - deserialize methods of this class, the :attr:`colander.null` value - will be returned. + If the :attr:`colander.null` value is passed to the serialize + method of this class, the :attr:`colander.null` value will be + returned. The subnodes of the :class:`colander.SchemaNode` that wraps this type are ignored. @@ -850,9 +824,6 @@ class Boolean(object): return appstruct and 'true' or 'false' def deserialize(self, node, cstruct): - if cstruct is null: - return null - try: result = str(cstruct) except: @@ -900,9 +871,9 @@ class GlobalObject(object): was supplied to the constructor, an :exc:`colander.Invalid` error will be raised. - If the :attr:`colander.null` value is passed to the serialize or - deserialize methods of this class, the :attr:`colander.null` value - will be returned. + If the :attr:`colander.null` value is passed to the serialize + method of this class, the :attr:`colander.null` value will be + returned. The subnodes of the :class:`colander.SchemaNode` that wraps this type are ignored. @@ -978,9 +949,6 @@ class GlobalObject(object): mapping={'val':appstruct}) ) def deserialize(self, node, cstruct): - if cstruct is null: - return null - if not isinstance(cstruct, basestring): raise Invalid(node, _('"${val}" is not a string', @@ -1033,9 +1001,9 @@ class DateTime(object): does so by using midnight of the day as the time, and uses the ``default_tzinfo`` to give the serialization a timezone. - If the :attr:`colander.null` value is passed to the serialize or - deserialize methods of this class, the :attr:`colander.null` value - will be returned. + If the :attr:`colander.null` value is passed to the serialize + method of this class, the :attr:`colander.null` value will be + returned. The subnodes of the :class:`colander.SchemaNode` that wraps this type are ignored. @@ -1065,9 +1033,6 @@ class DateTime(object): return appstruct.isoformat() def deserialize(self, node, cstruct): - if cstruct is null: - return null - try: result = iso8601.parse_date(cstruct) except (iso8601.ParseError, TypeError), e: @@ -1112,9 +1077,9 @@ class Date(object): time information related to the serialized value during deserialization. - If the :attr:`colander.null` value is passed to the serialize or - deserialize methods of this class, the :attr:`colander.null` value - will be returned. + If the :attr:`colander.null` value is passed to the serialize + method of this class, the :attr:`colander.null` value will be + returned. The subnodes of the :class:`colander.SchemaNode` that wraps this type are ignored. @@ -1138,9 +1103,6 @@ class Date(object): return appstruct.isoformat() def deserialize(self, node, cstruct): - if cstruct is null: - return null - try: result = iso8601.parse_date(cstruct) result = result.date() @@ -1169,19 +1131,18 @@ class SchemaNode(object): node are not known at construction time, they can later be added via the ``add`` method. - The constructor accepts these keyword arguments (via **kw): + The constructor accepts these keyword arguments: - ``name``: The name of this node. - ``default``: The default serialization value for this node. - Default: N/A (optional). If it is not provided, this node has - no default value and it will be considered 'serialization - required' (the ``srequired`` attribute will be ``True``). + Default: :attr:`colander.null`. - ``missing``: The default deserialization value for this node. - If it is not provided, this node has no missing value and it - will be considered 'required' (the ``required`` attribute will - be ``True``). + If it is not provided, the missing value of this node will be + :attr:`colander.null`, indicating that it is considered + 'required' (the ``required`` computed attribute will be + ``True``). - ``validator``: Optional validator for this node. It should be an object that implements the @@ -1192,7 +1153,7 @@ class SchemaNode(object): by Colander itself). - ``description``: The description for this node. Defaults to - ``''`` (the emtpty string). The description is used by + ``''`` (the empty string). The description is used by higher-level systems (not by Colander itself). """ @@ -1207,8 +1168,8 @@ class SchemaNode(object): def __init__(self, typ, *children, **kw): self.typ = typ self.validator = kw.pop('validator', None) - self.default = kw.pop('default', _marker) - self.missing = kw.pop('missing', _marker) + self.default = kw.pop('default', null) + self.missing = kw.pop('missing', null) self.name = kw.pop('name', '') self.title = kw.pop('title', self.name.capitalize()) self.description = kw.pop('description', '') @@ -1224,87 +1185,64 @@ class SchemaNode(object): self.name, ) - - @property - def srequired(self): - """ A property which returns ``True`` if a usable value - corresponding to this node is required to be present in a data - structure we're asked to serialize. A return value of - ``True`` implies that a usable ``default`` value wasn't - specified for this node. A return value of ``False`` implies - that a usable ``default`` value *was* specified for this node.""" - return self.default is _marker - @property def required(self): - """ A property which returns ``True`` if a usable value - corresponding to this node is required to be present in a data - structure we're asked to deserialize. A return value of - ``True`` implies that a usable ``missing`` value wasn't - specified for this node. A return value of ``False`` implies - that a usable ``missing`` value *was* specified for this node.""" - return self.missing is _marker + """ A property which returns ``True`` if the ``missing`` value + related to this node is the :attr:`colander.null` sentinel + value. - def serialize(self, appstruct=default): + A return value of ``True`` implies that a ``missing`` value + wasn't specified for this node. A return value of ``False`` + implies that a ``missing`` value was specified for this node.""" + return self.missing is null + + def serialize(self, appstruct=null): """ Serialize the :term:`appstruct` to a :term:`cstruct` based on the schema represented by this node and return the cstruct. - If ``appstruct`` is :attr:`colander.default`, do something - special: - - - If the ``default`` attribute of this node has been set, return - the serialized value of the ``default`` attribute. - - - If the ``default`` attribute of this node has not been set, - return the serialization of :attr:`colander.null`. + If ``appstruct`` is :attr:`colander.null`, return the + serialized value of this node's ``default`` attribute (by + default, the serialization of :attr:`colander.null`). If an ``appstruct`` argument is not explicitly provided, it - defaults to :attr:`colander.default`. + defaults to :attr:`colander.null`. """ - if appstruct is default: + if appstruct is null: appstruct = self.default - if appstruct is _marker: - # We cannot just return null here; we need to allow - # the node to serialize null to what it believes null - # should mean - appstruct = null cstruct = self.typ.serialize(self, appstruct) return cstruct - def deserialize(self, cstruct=default): + def deserialize(self, cstruct=null): """ Deserialize and validate the :term:`cstruct` into an :term:`appstruct` based on the schema, and return the deserialized, validated appstruct. If the cstruct cannot be validated, a :exc:`colander.Invalid` exception will be raised. - If ``cstruct`` is :attr:`colander.default`, do something special: + If ``cstruct`` is :attr:`colander.null`, do something special: - - If the ``missing`` attribute of this node has been set, - return it. + - If the ``missing`` attribute of this node has been set + explicitly, return its value. No deserialization or + validation of this value is performed; it is simply + returned. - - If the ``missing`` attribute of this node has not been set, - raise a :exc:`colander.Invalid` exception error. + - If the ``missing`` attribute of this node has not been set + explicitly, raise a :exc:`colander.Invalid` exception error. If a ``cstruct`` argument is not explicitly provided, it - defaults to :attr:`colander.default`. - - When used as a cstruct, :attr:`colander.null` is never passed to - a validator: it is considered intrinsically valid. + defaults to :attr:`colander.null`. """ - if cstruct is default: + if cstruct is null: appstruct = self.missing - if appstruct is _marker: + if appstruct is null: raise Invalid(self, _('Required')) - # We never validate the missing value + # We never deserialize or validate the missing value return appstruct appstruct = self.typ.deserialize(self, cstruct) - if appstruct is not null: - # We never validate the null value. - if self.validator is not None: - self.validator(self, appstruct) + if self.validator is not None: + self.validator(self, appstruct) return appstruct def add(self, node): diff --git a/colander/interfaces.py b/colander/interfaces.py index 27a0d3d..82ee077 100644 --- a/colander/interfaces.py +++ b/colander/interfaces.py @@ -1,54 +1,47 @@ -def Validator(struct, value): +def Validator(node, value): """ A validator is called after deserialization of a value. If ``value`` is not valid, raise a :class:`colander.Invalid` instance as an exception after. - ``struct`` is a :class:`colander.Structure` instance which - contains, among other things, the default value, the name of the - value, and a ``required`` flag indicating whether this value is - required. It is often ignored in simple validators. + ``node`` is a :class:`colander.SchemaNode` instance, for use when + raising a :class:`colander.Invalid` exception. """ class Type(object): - def serialize(self, struct, value): + def serialize(self, node, appstruct): """ - Serialize the object represented by ``value`` to a - data structure. The serialization should be composed of one or - more objects which can be deserialized by the + Serialize the :term:`appstruct` represented by ``appstruct`` + to a :term:`cstruct`. The serialization should be composed of + one or more objects which can be deserialized by the :meth:`colander.interfaces.Type.deserialize` method of this type. - This method should also do type validation of ``value``. + ``node`` is a :class:`colander.SchemaNode` instance. - ``struct`` is a :class:`colander.Structure` instance which - contains, among other things, the default value, the name of - the value, and a ``required`` flag indicating whether this - value is required. + ``appstruct`` is an :term:`appstruct`. - If the object cannot be serialized, or type validation for - ``value`` fails, a :exc:`colander.Invalid` exception should be - raised. + If ``appstruct`` is the special value :attr:`colander.null`, + the type should serialize a null value. + + If the object cannot be serialized for any reason, a + :exc:`colander.Invalid` exception should be raised. """ - def deserialize(self, struct, value): + def deserialize(self, node, cstruct): """ - Deserialze the serialization represented by ``value`` to a - data structure. The deserialization should be composed of one - or more objects which can be serialized by the + Deserialze the :term:`cstruct` represented by ``cstruct`` to + an :term:`appstruct`. The deserialization should be composed + of one or more objects which can be serialized by the :meth:`colander.interfaces.Type.serialize` method of this type. - This method should also do type validation of ``value``. + ``node`` is a :class:`colander.SchemaNode` instance. - ``struct`` is a :class:`colander.Structure` instance which - contains, among other things, the default value, the name of - the value, and a ``required`` flag indicating whether this - value is required. + ``cstruct`` is a :term:`cstruct`. - If the object cannot be deserialized, or type validation for - ``value`` fails, a :exc:`colander.Invalid` exception should be - raised. + If the object cannot be deserialized for any reason, a + :exc:`colander.Invalid` exception should be raised. """ diff --git a/colander/tests.py b/colander/tests.py index 0be376e..573eadb 100644 --- a/colander/tests.py +++ b/colander/tests.py @@ -326,13 +326,6 @@ class TestMapping(unittest.TestCase): except ValueError, e: # pragma: no cover raise AssertionError(e) - def test_deserialize_null(self): - from colander import null - node = DummySchemaNode(None) - typ = self._makeOne() - result = typ.deserialize(node, null) - self.assertEqual(result, null) - def test_deserialize_not_a_mapping(self): node = DummySchemaNode(None) typ = self._makeOne() @@ -353,14 +346,6 @@ class TestMapping(unittest.TestCase): result = typ.deserialize(node, {'a':1}) self.assertEqual(result, {'a':1}) - def test_deserialize_value_is_null(self): - node = DummySchemaNode(None) - from colander import null - node.children = [DummySchemaNode(None, name='a')] - typ = self._makeOne() - result = typ.deserialize(node, null) - self.assertEqual(result, null) - def test_deserialize_unknown_raise(self): node = DummySchemaNode(None) node.children = [DummySchemaNode(None, name='a')] @@ -397,7 +382,7 @@ class TestMapping(unittest.TestCase): ] typ = self._makeOne() result = typ.deserialize(node, {'a':1}) - self.assertEqual(result, {'a':1, 'b':colander.default}) + self.assertEqual(result, {'a':1, 'b':colander.null}) def test_serialize_null(self): import colander @@ -439,24 +424,16 @@ class TestMapping(unittest.TestCase): def test_serialize_value_is_null(self): node = DummySchemaNode(None) from colander import null - from colander import default node.children = [DummySchemaNode(None, name='a')] typ = self._makeOne() result = typ.serialize(node, null) - self.assertEqual(result, {'a':default}) + self.assertEqual(result, {'a':null}) class TestTuple(unittest.TestCase): def _makeOne(self): from colander import Tuple return Tuple() - def test_deserialize_null(self): - import colander - node = DummySchemaNode(None) - typ = self._makeOne() - result = typ.deserialize(node, colander.null) - self.assertEqual(result, colander.null) - def test_deserialize_not_iterable(self): node = DummySchemaNode(None) typ = self._makeOne() @@ -573,13 +550,6 @@ class TestSequence(unittest.TestCase): from colander import Sequence self.assertEqual(Seq, Sequence) - def test_deserialize_null(self): - import colander - node = DummySchemaNode(None) - typ = self._makeOne() - result = typ.deserialize(node, colander.null) - self.assertEqual(result, colander.null) - def test_deserialize_not_iterable(self): node = DummySchemaNode(None) typ = self._makeOne() @@ -682,13 +652,6 @@ class TestString(unittest.TestCase): result = typ.deserialize(node, '') self.assertEqual(result, '') - def test_deserialize_null(self): - from colander import null - node = DummySchemaNode(None) - typ = self._makeOne(None) - result = typ.deserialize(node, null) - self.assertEqual(result, null) - def test_deserialize_uncooperative(self): val = Uncooperative() node = DummySchemaNode(None) @@ -816,14 +779,6 @@ class TestInteger(unittest.TestCase): result = typ.deserialize(node, val) self.assertEqual(result, 1) - def test_deserialize_null(self): - import colander - val = colander.null - node = DummySchemaNode(None) - typ = self._makeOne() - result = typ.deserialize(node, val) - self.assertEqual(result, colander.null) - def test_serialize_fails(self): val = 'P' node = DummySchemaNode(None) @@ -872,14 +827,6 @@ class TestFloat(unittest.TestCase): result = typ.deserialize(node, val) self.assertEqual(result, 1.0) - def test_deserialize_null(self): - import colander - val = colander.null - node = DummySchemaNode(None) - typ = self._makeOne() - result = typ.deserialize(node, val) - self.assertEqual(result, colander.null) - def test_serialize_fails(self): val = 'P' node = DummySchemaNode(None) @@ -929,14 +876,6 @@ class TestDecimal(unittest.TestCase): result = typ.deserialize(node, val) self.assertEqual(result, decimal.Decimal('1.0')) - def test_deserialize_null(self): - import colander - val = colander.null - node = DummySchemaNode(None) - typ = self._makeOne() - result = typ.deserialize(node, val) - self.assertEqual(result, colander.null) - def test_serialize_fails(self): val = 'P' node = DummySchemaNode(None) @@ -984,14 +923,6 @@ class TestBoolean(unittest.TestCase): e = invalid_exc(typ.deserialize, node, Uncooperative()) self.failUnless(e.msg.endswith('not a string')) - def test_deserialize_null(self): - import colander - val = colander.null - node = DummySchemaNode(None) - typ = self._makeOne() - result = typ.deserialize(node, val) - self.assertEqual(result, colander.null) - def test_serialize(self): typ = self._makeOne() node = DummySchemaNode(None) @@ -1115,14 +1046,6 @@ class TestGlobalObject(unittest.TestCase): self.assertRaises(ImportError, typ._pkg_resources_style, None, ':notexisting') - def test_deserialize_null(self): - import colander - val = colander.null - node = DummySchemaNode(None) - typ = self._makeOne() - result = typ.deserialize(node, val) - self.assertEqual(result, colander.null) - def test_deserialize_not_a_string(self): typ = self._makeOne() node = DummySchemaNode(None) @@ -1237,14 +1160,6 @@ class TestDateTime(unittest.TestCase): expected = dt.isoformat() self.assertEqual(result, expected) - def test_deserialize_null(self): - import colander - val = colander.null - node = DummySchemaNode(None) - typ = self._makeOne() - result = typ.deserialize(node, val) - self.assertEqual(result, colander.null) - def test_deserialize_date(self): import datetime import iso8601 @@ -1318,14 +1233,6 @@ class TestDate(unittest.TestCase): expected = dt.date().isoformat() self.assertEqual(result, expected) - def test_deserialize_null(self): - import colander - val = colander.null - node = DummySchemaNode(None) - typ = self._makeOne() - result = typ.deserialize(node, val) - self.assertEqual(result, colander.null) - def test_deserialize_invalid_ParseError(self): node = DummySchemaNode(None) typ = self._makeOne() @@ -1400,10 +1307,6 @@ class TestSchemaNode(unittest.TestCase): node = self._makeOne(None, missing=1) self.assertEqual(node.required, False) - def test_srequired_false(self): - node = self._makeOne(None, default=1) - self.assertEqual(node.srequired, False) - def test_deserialize_no_validator(self): typ = DummyType() node = self._makeOne(typ) @@ -1417,35 +1320,19 @@ class TestSchemaNode(unittest.TestCase): e = invalid_exc(node.deserialize, 1) self.assertEqual(e.msg, 'Wrong') - def test_deserialize_value_is_default_no_missing(self): + def test_deserialize_value_is_null_no_missing(self): + from colander import null + from colander import Invalid typ = DummyType() node = self._makeOne(typ) - from colander import default - from colander import Invalid - self.assertRaises(Invalid, node.deserialize, default) + self.assertRaises(Invalid, node.deserialize, null) - def test_deserialize_value_is_default_with_missing(self): + def test_deserialize_value_is_null_with_missing(self): + from colander import null typ = DummyType() node = self._makeOne(typ) node.missing = 'abc' - from colander import default - self.assertEqual(node.deserialize(default), 'abc') - - def test_deserialize_value_is_default_with_missing_null(self): - from colander import null - from colander import default - typ = DummyType() - node = self._makeOne(typ) - node.missing = null - self.assertEqual(node.deserialize(default), null) - - def test_deserialize_value_is_null_validator_not_used(self): - from colander import null - typ = DummyType() - validator = DummyValidator(msg='Wrong') - node = self._makeOne(typ, validator=validator) - value = node.deserialize(null) - self.assertEqual(value, null) + self.assertEqual(node.deserialize(null), 'abc') def test_deserialize_noargs_uses_default(self): typ = DummyType() @@ -1459,20 +1346,19 @@ class TestSchemaNode(unittest.TestCase): result = node.serialize(1) self.assertEqual(result, 1) - def test_serialize_value_is_default_no_default(self): + def test_serialize_value_is_null_no_default(self): + from colander import null typ = DummyType() node = self._makeOne(typ) - from colander import default - from colander import null - result = node.serialize(default) + result = node.serialize(null) self.assertEqual(result, null) - def test_serialize_value_is_default_with_default(self): + def test_serialize_value_is_null_with_default(self): + from colander import null typ = DummyType() node = self._makeOne(typ) node.default = 1 - from colander import default - result = node.serialize(default) + result = node.serialize(null) self.assertEqual(result, 1) def test_serialize_noargs_uses_default(self): @@ -1725,12 +1611,6 @@ class TestDeclarative(unittest.TestCase, TestFunctional): schema = MainSchema() return schema -class Test_default(unittest.TestCase): - def test___repr__(self): - from colander import default - self.assertEqual(repr(default), '') - - class Test_null(unittest.TestCase): def test___nonzero__(self): from colander import null @@ -1740,11 +1620,6 @@ class Test_null(unittest.TestCase): from colander import null self.assertEqual(repr(null), '') -class Test__marker(unittest.TestCase): - def test___repr__(self): - from colander import _marker - self.assertEqual(repr(_marker), '') - class Dummy(object): pass diff --git a/docs/api.rst b/docs/api.rst index e7596cd..ccd7046 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -112,4 +112,5 @@ Schema-Related .. autoclass:: SequenceSchema + .. attribute:: null diff --git a/docs/basics.rst b/docs/basics.rst index 4ce5225..75ab5f7 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -109,11 +109,11 @@ serialization. It should be the deserialized representation. If a schema node does not have a default, it is considered "serialization required". -The *missing* of a schema node indicates the value to be deserialized -if a value for the schema node is not found in the input data during -deserialization. It should be the deserialized representation. If a -schema node does not have a default, it is considered "deserialization -required". +The *missing* of a schema node indicates the value if a value for the +schema node is not found in the input data during deserialization. It +should be the deserialized representation. If a schema node does not +have a default, it is considered "deserialization required". This +value is never validated; it is considered pre-validated. The *name* of a schema node appears in error reports. @@ -413,13 +413,13 @@ The value for ``serialized`` above will be ``{'age':'20', Serialization and deserialization are not completely symmetric, however. Although schema-driven data conversion happens during -serialization, and defaults are injected as necessary, :mod:`colander` -types are defined in such a way that structural validation and -validation of values does *not* happen as it does during -deserialization. For example, the :attr:`colander.null` value is -substituted for every missing subvalue in an appstruct, and none of -the validators associated with the schema or any of is nodes is -invoked. +serialization, and default values are injected as necessary, +:mod:`colander` types are defined in such a way that structural +validation and validation of values does *not* happen as it does +during deserialization. For example, the :attr:`colander.null` value +is substituted into the cstruct for every missing subvalue in an +appstruct, and none of the validators associated with the schema or +any of is nodes is invoked. This usually means you may "partially" serialize an appstruct where some of the values are missing. If we try to serialize partial data @@ -438,7 +438,7 @@ string, and the missing ``name`` attribute has been replaced with :attr:`colander.null`. Above, even though we did not include the ``name`` attribute in the appstruct we fed to ``serialize``, an error is *not* raised. For more information about :attr:`colander.null` -substitution during serialization, see :ref:`serializing_default`. +substitution during serialization, see :ref:`serializing_null`. The corollary: it is the responsibility of the developer to ensure he serializes "the right" data; :mod:`colander` will not raise an error diff --git a/docs/defaults.rst b/docs/defaults.rst deleted file mode 100644 index 55f86db..0000000 --- a/docs/defaults.rst +++ /dev/null @@ -1,405 +0,0 @@ -.. _default_and_null: - -Default and Null Values -======================= - -Two sentinel values have special meanings during serialization and -deserialization: :attr:`colander.default` and :attr:`colander.null`. -Both :attr:`colander.default` and :attr:`colander.null` are used as -sentinel values during the serialization and deserialization -processes, but they are not equivalent. Each represents a different -concept. - -:attr:`colander.default` is a sentinel value which may be passed to -:meth:`colander.SchemaNode.serialize` or to -:meth:`colander.SchemaNode.deserialize`. The use of -:attr:`colander.default` indicates that the value corresponding to the -node it's passed to is missing, and if possible, the *default value* -(during serialization, see :ref:`serializing_default`) or *missing -value* (during deserialization, see :ref:`deserializing_default`) for -the corresponding node should be used instead. - -.. note:: - - It makes sense for :attr:`colander.default` to be present in a data - structure passed to :meth:`colander.SchemaNode.serialize` or to - :meth:`colander.SchemaNode.deserialize` but it should never be - present in a schema definition and it should never be present in - the output of a serialization or deserialization. For example, it - is not reasonable to use the :attr:`colander.default` value itself - as the ``default`` or ``missing`` argument to a - :class:`colander.SchemaNode` constructor. Passing - :attr:`colander.default` as the ``default`` or ``missing`` - arguments to a schema node constructor will not do anything useful - (it is not explicitly prevented, it's just nonsensical). - :attr:`colander.default` should also never be present in the result - of serialization or the result of deserialization: it will only - ever be present in the input, never in the output. - -:attr:`colander.null` is a sentinel representing that the *null* value -for a corresponding node should be serialized (see -:ref:`serializing_null`) or deserialized (see -:ref:`deserializing_null`). The :attr:`colander.null` value may be -present directly in the data structure passed to -:meth:`colander.SchemaNode.serialize` or -:meth:`colander.SchemaNdoe.deserialize` but it is also not uncommon -for :attr:`colander.null` to be the *default value* (``default``) or -*missing value* (``missing``) for a node. - -.. note:: - - Unlike :attr:`colander.default`, :attr:`colander.null` is useful - both within the data structure passed to - :meth:`colander.SchemaNode.serialize` and - :meth:`colander.SchemaNode.deserialize` and within a schema - definition. - -.. _serializing_default_and_null: - -Serializing Default and Null Values ------------------------------------ - -It is possible to serialize both the default and null values. - -.. _serializing_default: - -Serializing The :attr:`colander.default` Value -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A node will attempt to serialize its *default value* during -:meth:`colander.SchemaNode.serialize` if a value it is provided is -*unspecified*. *Unspecified* means: - -#) The value expected by the schema is present in the data structure - passed to :meth:`colander.SchemaNode.serialize` but it is the - literal value :attr:`colander.default`. - -#) The value expected by the schema is a subkey of a mapping, but that - key is missing from the mapping in the data structure passed to - :meth:`colander.SchemaNode.serialize`: - -The *default value* of a node is specified during schema creation as -its ``default`` attribute / argument. For example, the ``hair_color`` -node below has a default value of ``brown``: - -.. code-block:: python - - import colander - - class Person(colander.MappingSchema): - name = colander.SchemaNode(colander.String()) - age = colander.SchemaNode(colander.Int(), - validator=colander.Range(0, 200)) - hair_color = colander.SchemaNode(colander.String(), default='brown') - -Because the ``hair_color`` node is passed a ``default`` value, if the -above schema is used to serialize a mapping that does not have a -``hair_color`` key, the default will be serialized: - -.. code-block:: python - - schema = Person() - serialized = schema.serialize({'name':'Fred', 'age':20}) - -Even though we did not include the ``hair_color`` attribute in the -data we fed to ``serialize``, the value of ``serialized`` above will -be ``{'name':'Fred, 'age':'20', 'hair_color':'brown'}``. This is due -to the ``default`` value provided during schema node construction for -``hair_color``. - -The same outcome would have been true had we fed the schema a mapping -for serialization which had the :attr:`colander.default` sentinel as -the ``hair_color`` value: - -.. code-block:: python - - from colander import default - schema = Person() - serialized = schema.serialize({'name':'Fred', 'age':20, - 'hair_color':default}) - -In the above, the value of ``serialized`` above will be -``{'name':'Fred, 'age':'20', 'hair_color':'brown'}`` just as it was in -the example where ``hair_color`` was not present in the mapping. - -On the other hand, if the ``hair_color`` value is missing or -:attr:`colander.default`, and the schema does *not* name a ``default`` -value for ``hair_color``, it will be present in the resulting -serialization as :attr:`colander.null`: - -.. code-block:: python - - import colander - - class Person(colander.MappingSchema): - name = colander.SchemaNode(colander.String()) - age = colander.SchemaNode(colander.Int(), - validator=colander.Range(0, 200)) - hair_color = colander.SchemaNode(colander.String()) - - - schema = Person() - serialized = schema.serialize({'name':'Fred', 'age':20}) - -The value for ``serialized`` above will be ``{'name':'Fred, -'age':'20', 'hair_color':colander.null}``. We did not include the -``hair_color`` attribute in the data we fed to ``serialize``, and -there was no ``default`` value associated with ``hair_color`` to fall -back to, so the :attr:`colander.null` value is used in the resulting -serialization. - -Serializations can be done of partial data structures; the -:attr:`colander.null` value is inserted into the serialization -whenever a corresponding value in the data structure being serialized -is missing. - -.. note:: The injection of the :attr:`colander.null` value into a - serialization when a default doesn't exist for the corresponding - node is not a behavior shared during both serialization and - deserialization. While a *serialization* can be performed against - a partial data structure without corresponding node defaults, a - *deserialization* cannot be done to partial data without - corresponding node ``missing`` values. When a value is missing - from a data structure being deserialized, and no ``missing`` value - exists for the node corresponding to the missing item in the data - structure, a :class:`colander.Invalid` exception will be the - result. - -.. _serializing_null: - -Serializing The :attr:`colander.null` Value -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The value :attr:`colander.null` has special meaning to types during -serialization. If :attr:`colander.null` is used as the serialization -value passed to a type, it signals that the type should serialize a -type-specific *null value*. - -Serialization of a *null value* is completely type-specific, meaning -each type is free to serialize :attr:`colander.null` to a value that -makes sense for that particular type. For example, the null -serialization value of a :class:`colander.String` type is the empty -string. - -The :attr:`colander.null` value will be passed to a type either -directly or indirectly: - -- directly: because :attr:`colander.null` is passed directly to the - ``serialize`` method of a node. - -- indirectly: because a node uses a :attr:`colander.null` value as its - ``default`` attribute and the value passed to the serialize method - of a node is missing or :attr:`colander.default` (see - :ref:`serializing_default_and_null`). - -When a particular type cannot serialize the null value to anything -sensible, the type's serialize method must return the null object -itself as a serialization. For example, when the -:class:`colander.Boolean` type is asked to serialize the -:attr:`colander.null` value, its ``serialize`` method simply returns -the :attr:`colander.null` value (because null is conceptually neither -true nor false). Therefore, when :attr:`colander.null` is used as -input to serialization, or as the default value of a schema node, it -is possible that the :attr:`colander.null` value will placed into the -serialized data structure. The consumer of the serialization must -anticipate this and deal with the special :attr:`colander.null` value -in the output however it sees fit. - -Here's an example of a serialization which will have the sentinel -value :attr:`colander.null` in the serialized output: - -.. code-block:: python - - import colander - - class Person(colander.MappingSchema): - name = colander.SchemaNode(colander.String()) - age = colander.SchemaNode(colander.Int(), default=colander.null) - -Because the ``age`` node is passed a ``default`` value of -:attr:`colander.null`, if the above schema is used to serialize a -mapping that does not have an ``age`` key, the default will be -serialized into the output: - -.. code-block:: python - - schema = Person() - serialized = schema.serialize({'name':'Fred'}) - -The value for ``serialized`` above will be ``{'name':'Fred, -'age':colander.null}``. We did not include the ``age`` attribute in -the data we fed to ``serialize``, but there was a ``default`` value -associated with ``age`` to fall back to: :attr:`colander.null`. -However, the :class:`colander.Int` type cannot serialize null to any -*particular* integer, so it returns the :attr:`colander.null` object -itself. As a result, the raw :attr:`colander.null` value is simply -injected into the resulting serialization. The caller of the -:meth:`colander.SchemaNode.serialize` method will need to deal with -this value appropriately. - -Serialization Combinations -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To reduce the potential for confusion about the difference between -:attr:`colander.default` and :attr:`colander.null` during -serialization, here's a table of serialization combinations. Within -this table, the ``Value`` column represents the value passed to the -:meth:`colander.SchemaNode.serialize` method of a particular schema -node, the ``Default`` column represents the ``default`` value of that -schema node, and the ``Result`` column is a description of the result -of invoking the :meth:`colander.SchemaNode.serialize` method of the -schema node with the effective value. - -===================== ===================== =========================== -Value Default Result -===================== ===================== =========================== -colander.default Invalid exception raised - Invalid exception raised -colander.default value value serialized - value value serialized -colander.default colander.null null serialized - colander.null null serialized -value value serialized -value_a value_b value_a serialized -value colander.null value serialized -colander.null null serialized -colander.null value null serialized -colander.null colander.null null serialized -===================== ===================== =========================== - -.. note:: ```` in the above table represents the circumstance - in which a key present in a :class:`colander.MappingSchema` is not - present in a mapping passed to its - :meth:`colander.SchemaNode.serialize` method. In reality, - ```` means exactly the same thing as - :attr:`colander.default`, because the :class:`colander.Mapping` - type does the equivalent of ``mapping.get(keyname, - colander.default)`` to find a subvalue during serialization. - -.. _deserializing_default_and_null: - -Deserializing Default and Null Values -------------------------------------- - -It is possible to deserialize both the default and null values. - -.. _deserializing_default: - -Deserializing The :attr:`colander.default` Value -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The data structure passed to :meth:`colander.SchemaNode.deserialize` -may contain one or more :attr:`colander.default` sentinel markers. - -When a :attr:`colander.default` sentinel marker is passed to the -:meth:`colander.SchemaNode.deserialize` method of a particular node in -a schema, the node will take the following steps: - -- If the schema node has a valid ``missing`` attribute (the node's - constructor was supplied with a ``missing`` argument), the - ``missing`` value will be returned. Note that when this happens, - the ``missing`` value is not validated by any schema node validator: - it is simply returned. - -- If the schema node does *not* have a valid ``missing`` attribute - (the node's constructor was not supplied with a ``missing`` value), - a :exc:`colander.Invalid` exception will be raised with a message - indicating that the field is required. - -.. note:: There are differences between serialization and - deserialization involving the :attr:`colander.default` value. - During serialization, if an :attr:`colander.default` value is - encountered, and no valid ``default`` attribute exists on the node - related to the value, a :attr:`colander.null` attribute is - returned. The the first difference: deserialization doesn't use - the ``default`` attribute of the node to find a default value in - the same circumstance; instead it uses the ``missing`` attribute. - The second difference: if, during deserialization, an - :attr:`colander.default` value is encountered as the value passed - to the deserialize method, and no valid ``missing`` value exists - for the node, a :exc:`colander.Invalid` exception is raised - (:attr:`colander.null` is not returned, as it is during - serialization). - -.. _deserializing_null: - -Deserializing The :attr:`colander.null` Value -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The value :attr:`colander.null` has special meaning to types during -deserialization. If :attr:`colander.null` is used as a -deserialization value to a type, it signals that the type should -deserialize the type-specific *null value*. - -Deserialization of a *null value* is completely type-specific, meaning -each type is free to deserialize :attr:`colander.null` to a value that -makes sense for that particular type. For example, the -deserialization of a :class:`colander.String` type is the empty -string. - -The :attr:`colander.null` value will be passed to a type either -directly or indirectly: - -- directly: because :attr:`colander.null` is passed directly to the - ``deserialize`` method of a node. - -- indirectly: because a node uses a :attr:`colander.null` value as its - ``missing`` attribute and the value passed to the serialize method - of a node is missing or :attr:`colander.default`. - -When a particular type cannot deserialize the null value to anything -sensible, the type's deserialize method must return the null object -itself as a serialization. - -For example, when the :class:`colander.Boolean` type is asked to -deserialize the :attr:`colander.null` value, its ``deserialize`` -method simply returns the :attr:`colander.null` value (because null is -conceptually neither true nor false). Therefore, when -:attr:`colander.null` is used as input to deserialization, or as the -``missing`` value of a schema node, it is possible that the -:attr:`colander.null` value will be placed into the deserialized data -structure. The consumer of the deserialization must anticipate this -and deal with the special :attr:`colander.null` value in the output -however it sees fit. - -Note that deserialization of the null value never invokes a validator. - -Deserialization Combinations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To reduce the potential for confusion about the difference between -:attr:`colander.default` and :attr:`colander.null` during -deserialization, here's a table of serialization combinations. Within -this table, the ``Value`` column represents the value passed to the -:meth:`colander.SchemaNode.deserialize` method of a particular schema -node, the ``Missing`` column represents the ``missing`` value of that -schema node, and the ``Result`` column is a description of the result -of invoking the :meth:`colander.SchemaNode.deserialize` method of the -schema node with the effective value. - -===================== ===================== =========================== -Value Missing Result -===================== ===================== =========================== -colander.default Invalid exception raised - Invalid exception raised -colander.default value value deserialized - value value deserialized -colander.default colander.null null deserialized - colander.null null deserialized -value value deserialized -value_a value_b value_a deserialized -value colander.null value deserialized -colander.null null deserialized -colander.null value null deserialized -colander.null colander.null null deserialized -===================== ===================== =========================== - -.. note:: ```` in the above table represents the circumstance - in which a key present in a :class:`colander.MappingSchema` is not - present in a mapping passed to its - :meth:`colander.SchemaNode.deserialize` method. In reality, - ```` means exactly the same thing as - :attr:`colander.default`, because the :class:`colander.Mapping` - type does the equivalent of ``mapping.get(keyname, - colander.default)`` to find a subvalue during deserialization. - diff --git a/docs/extending.rst b/docs/extending.rst index 5eef242..116ea04 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -1,8 +1,10 @@ Extending Colander ================== -You can extend Colander by defining a new type or defining a new -validator. +You can extend Colander by defining a new :term:`type` or by defining +a new :term:`validator`. + +.. _defining_a_new_type: Defining a New Type ------------------- @@ -15,12 +17,10 @@ Python data structure (a :term:`appstruct`). Here's a type which implements boolean serialization and deserialization. It serializes a boolean to the string ``true`` or -``false``; it deserializes a string (presumably ``true`` or ``false``, -but allows some wiggle room for ``t``, ``on``, ``yes``, ``y``, and -``1``) to a boolean value. It deals with the sentinel value -:attr:`colander.null` by simply returning it when asked to serialize -or deserialize it: during serialization, the caller will need to -anticipate this. +``false`` or the special :attr:`colander.null` sentinel; it then +deserializes a string (presumably ``true`` or ``false``, but allows +some wiggle room for ``t``, ``on``, ``yes``, ``y``, and ``1``) to a +boolean value. .. code-block:: python :linenos: @@ -36,8 +36,6 @@ anticipate this. return appstruct and 'true' or 'false' def deserialize(self, node, cstruct): - if cstruct is null: - return null if not isinstance(cstruct, basestring): raise Invalid(node, '%r is not a string' % cstruct) value = cstruct.lower() @@ -45,6 +43,13 @@ anticipate this. return True return False +Note that the ``deserialize`` method of a type does not need to +explicitly deserialize the :attr:`colander.null` value. +Deserialization of the null value is dealt with at a higher level +(with the :meth:`colander.SchemaNode.deserialize` method); a type will +never receive an :attr:`colander.null` value as a ``cstruct`` argument +to its ``deserialize`` method. + Here's how you would use the resulting class as part of a schema: .. code-block:: python @@ -61,14 +66,16 @@ defined in the ``Boolean`` type class. Note that the only two real constraints of a type class are: -- its ``serialize`` method must be able to make sense of a value - generated by its ``deserialize`` method and vice versa. - - it must deal specially with the value :attr:`colander.null` within - both ``serialize`` and ``deserialize``, either returning it or - translating it to a type-specific null value. + ``serialize``, translating it to a type-specific null value. -The serialize and method of a type accept two values: ``node``, and +- its ``serialize`` method must be able to make sense of a value + generated by its ``deserialize`` method and vice versa, except that + the ``deserialize`` method needn't deal with the + :attr:`colander.null` value specially even if the ``serialize`` + method returns it. + +The ``serialize`` method of a type accepts two values: ``node``, and ``appstruct``. ``node`` will be the schema node associated with this type. It is used when the type must raise a :exc:`colander.Invalid` error, which expects a schema node as its first constructor argument. @@ -82,9 +89,22 @@ error, which expects a schema node as its first constructor argument. ``cstruct`` will be the :term:`cstruct` value that needs to be deserialized. +A type class does not need to implement a constructor (``__init__``), +but it isn't prevented from doing so if it needs to accept arguments; +Colander itself doesn't construct any types, only users of Colander +schemas do, so how types are constructed is beyond the scope of +Colander itself. + +The :exc:`colander.Invalid` exception may be raised during +serialization or deserialization as necessary for whatever reason the +type feels appropriate (the inability to serialize or deserialize a +value being the most common case). + For a more formal definition of a the interface of a type, see :class:`colander.interfaces.Type`. +.. _defining_a_new_validator: + Defining a New Validator ------------------------ diff --git a/docs/glossary.rst b/docs/glossary.rst index 12bd15d..72db9aa 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -12,22 +12,37 @@ Glossary consumed by the :meth:`colander.SchemaNode.deserialize` method. appstruct - A raw application data structure (complex Python objects). + A raw application data structure (a structure of complex Python + objects), passed to the :meth:`colander.SchemaNode.serialize` + method for serialization. The + :meth:`colander.SchemaNode.deserialize` method accepts a + :term:`cstruct` and returns an appstruct. schema A nested collection of :term:`schema node` objects representing an arrangement of data. schema node - A schema node can serialize an :term:`appstruct` to a - :term:`cstruct` and deserialize a :term:`cstruct` to an - :term:`appstruct` (object derived from + A schema node is an object which can serialize an + :term:`appstruct` to a :term:`cstruct` and deserialize a + :term:`appstruct` from a :term:`cstruct` an (object derived from :class:`colander.SchemaNode` or one of the colander Schema classes). + type + An object representing a particular type of data (mapping, + boolean, string, etc) capable of serializing an :term:`appstruct` + and of deserializing a :term:`cstruct`. Colander has various + built-in types (:class:`colander.String`, + :class:`colander.Mapping`, etc) and may be extended with + additional types (see :ref:`defining_a_new_type`). + validator - A Colander validator callable. Accepts a ``node`` - object and a ``value`` and either raises an - :exc:`colander.Invalid` exception or returns ``None``. Used as - the ``validator=`` argument to a schema node, ensuring that the - input meets the requirements of the schema. + A Colander validator callable. Accepts a ``node`` object and a + ``value`` and either raises an :exc:`colander.Invalid` exception + or returns ``None``. Used as the ``validator=`` argument to a + schema node, ensuring that the input meets the requirements of + the schema. Built-in validators exist in Colander + (e.g. :class:`colander.OneOf`, :class:`colander.Range`, etc), and + new validators can be defined to extend Colander (see + :ref:`defining_a_new_validator`). diff --git a/docs/index.rst b/docs/index.rst index 2c36d99..35655b6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -54,7 +54,7 @@ internationalizable. :maxdepth: 2 basics.rst - defaults.rst + null.rst extending.rst interfaces.rst api.rst diff --git a/docs/null.rst b/docs/null.rst new file mode 100644 index 0000000..823c0f4 --- /dev/null +++ b/docs/null.rst @@ -0,0 +1,306 @@ +.. _null: + +The Null Value +============== + +:attr:`colander.null` is a sentinel value which may be passed to +:meth:`colander.SchemaNode.serialize` during serialization or to +:meth:`colander.SchemaNode.deserialize` during deserialization. + +During serialization, the use of :attr:`colander.null` indicates that +the :term:`appstruct` value corresponding to the node it's passed to +is missing and the value of the ``default`` attribute of the +corresponding node should be used instead. + +During deserialization, the use of :attr:`colander.null` indicates +that the :term:`cstruct` value corresponding to the node it's passed +to is missing, and if possible, the value of the ``missing`` attribute +of the corresponding node should be used instead. + +Note that :attr:`colander.null` has no relationship to the built-in +Python ``None`` value. + +.. _serializing_null: + +Serializing The Null Value +-------------------------- + +A node will attempt to serialize its *default value* during +:meth:`colander.SchemaNode.serialize` if the value it is passed as an +``appstruct`` argument is the :attr:`colander.null` sentinel value. + +The *default value* of a node is specified during schema creation as +its ``default`` attribute / argument. For example, the ``hair_color`` +node below has a default value of ``brown``: + +.. code-block:: python + + import colander + + class Person(colander.MappingSchema): + name = colander.SchemaNode(colander.String()) + age = colander.SchemaNode(colander.Int(), + validator=colander.Range(0, 200)) + hair_color = colander.SchemaNode(colander.String(), default='brown') + +Because the ``hair_color`` node is passed a ``default`` value, if the +above schema is used to serialize a mapping that does not have a +``hair_color`` key, the default will be serialized: + +.. code-block:: python + + schema = Person() + serialized = schema.serialize({'name':'Fred', 'age':20}) + +Even though we did not include the ``hair_color`` attribute in the +appstruct we fed to ``serialize``, the value of ``serialized`` above +will be ``{'name':'Fred, 'age':'20', 'hair_color':'brown'}``. This is +because a ``default`` value of ``brown`` was provided during schema +node construction for ``hair_color``. + +The same outcome would have been true had we fed the schema a mapping +for serialization which had the :attr:`colander.null` sentinel as the +``hair_color`` value: + +.. code-block:: python + + import colander + + schema = Person() + serialized = schema.serialize({'name':'Fred', 'age':20, + 'hair_color':colander.null}) + +When the above is run, the value of ``serialized`` will be +``{'name':'Fred, 'age':'20', 'hair_color':'brown'}`` just as it was in +the example where ``hair_color`` was not present in the mapping. + +As we can see, serializations may be done of partial data structures; +the :attr:`colander.null` value is inserted into the serialization +whenever a corresponding value in the data structure being serialized +is missing. + +.. note:: The injection of the :attr:`colander.null` value into a + serialization when a default doesn't exist for the corresponding + node is not a behavior shared during both serialization and + deserialization. While a *serialization* can be performed against + a partial data structure without corresponding node defaults, a + *deserialization* cannot be done to partial data without + corresponding node ``missing`` values. When a value is missing + from a data structure being deserialized, and no ``missing`` value + exists for the node corresponding to the missing item in the data + structure, a :class:`colander.Invalid` exception will be the + result. + +If, during serialization, a value for the node is missing from the +cstruct and the node does not possess an explicit *default value*, the +:attr:`colander.null` sentinel value is passed to the type's +``serialize`` method directly, instructing the type to serialize a +type-specific *null value*. + +Serialization of a null value is completely type-specific, meaning +each type is free to serialize :attr:`colander.null` to a value that +makes sense for that particular type. For example, the null +serialization value of a :class:`colander.String` type is the empty +string. + +For example: + +.. code-block:: python + + import colander + + class Person(colander.MappingSchema): + name = colander.SchemaNode(colander.String()) + age = colander.SchemaNode(colander.Int(), + validator=colander.Range(0, 200)) + hair_color = colander.SchemaNode(colander.String()) + + + schema = Person() + serialized = schema.serialize({'name':'Fred', 'age':20}) + +In the above example, the ``hair_color`` value is missing and the +schema does *not* name a ``default`` value for ``hair_color``. +However, when we attempt to serialize the data structure, an error is +not raised. Instead, the value for ``serialized`` above will be +``{'name':'Fred, 'age':'20', 'hair_color':colander.null}``. + +Because we did not include the ``hair_color`` attribute in the data we +fed to ``serialize``, and there was no ``default`` value associated +with ``hair_color`` to fall back to, the :attr:`colander.null` value +is passed as the ``appstruct`` value to the ``serialize`` method of +the underlying type (:class:`colander.String`). The return value of +that type's ``serialize`` method when :attr:`colander.null` is passed +as the ``appstruct`` is placed into the serialization. +:class:`colander.String` happens to *return* :attr:`colander.null` +when it is passed :attr:`colander.null` as its appstruct argument, so +this is what winds up in the resulting cstruct. + +The :attr:`colander.null` value will be passed to a type either +directly or indirectly: + +- directly: because :attr:`colander.null` is passed directly to the + ``serialize`` method of a node. + +- indirectly: because every schema node uses a :attr:`colander.null` + value as its ``default`` attribute when no explicit default is + provided. + +When a particular type cannot serialize the null value to anything +sensible, that type's ``serialize`` method must return the null object +itself as a serialization. For example, when the +:class:`colander.Boolean` type is asked to serialize the +:attr:`colander.null` value, its ``serialize`` method simply returns +the :attr:`colander.null` value (because null is conceptually neither +true nor false). + +Therefore, when :attr:`colander.null` is used as input to +serialization, or as the default value of a schema node, it is +possible that the :attr:`colander.null` value will placed into the +serialized data structure. The consumer of the serialization must +anticipate this and deal with the special :attr:`colander.null` value +in the output however it sees fit. + +Serialization Combinations +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Within this table, the ``Value`` column represents the value passed to +the :meth:`colander.SchemaNode.serialize` method of a particular +schema node, the ``Default`` column represents the ``default`` value +of that schema node, and the ``Result`` column is a description of the +result of invoking the :meth:`colander.SchemaNode.serialize` method of +the schema node with the effective value. + +===================== ===================== =========================== +Value Default Result +===================== ===================== =========================== +colander.null value value serialized + value value serialized +colander.null colander.null null serialized + colander.null null serialized +value value serialized +value_a value_b value_a serialized +value colander.null value serialized +colander.null null serialized +colander.null value null serialized +===================== ===================== =========================== + +.. note:: ```` in the above table represents the circumstance + in which a key present in a :class:`colander.MappingSchema` is not + present in a mapping passed to its + :meth:`colander.SchemaNode.serialize` method. In reality, + ```` means exactly the same thing as + :attr:`colanderr.null`, because the :class:`colander.Mapping` type + does the equivalent of ``mapping.get(keyname, colander.null)`` to + find a subvalue during serialization. + +.. _deserializing_null: + +Deserializing The Null Value +---------------------------- + +The data structure passed to :meth:`colander.SchemaNode.deserialize` +may contain one or more :attr:`colander.null` sentinel markers. + +When a :attr:`colander.null` sentinel marker is passed to the +:meth:`colander.SchemaNode.deserialize` method of a particular node in +a schema, the node will take the following steps: + +- If the schema node has an explicit ``missing`` attribute (the node's + constructor was supplied with an explicit ``missing`` argument), the + ``missing`` value will be returned. Note that when this happens, + the ``missing`` value is not validated by any schema node validator: + it is simply returned. + +- If the schema node does *not* have an explicitly provided + ``missing`` attribute (the node's constructor was not supplied with + an explicit ``missing`` value), a :exc:`colander.Invalid` exception + will be raised with a message indicating that the field is required. + +.. note:: There are differences between serialization and + deserialization involving the :attr:`colander.null` value. During + serialization, if an :attr:`colander.null` value is encountered, + and no valid ``default`` attribute exists on the node related to + the value the *null value* for that node is returned. + Deserialization, however, doesn't use the ``default`` attribute of + the node to find a default deserialization value in the same + circumstance; instead it uses the ``missing`` attribute instead. + Also, if, during deserialization, an :attr:`colander.null` value is + encountered as the value passed to the deserialize method, and no + explicit ``missing`` value exists for the node, a + :exc:`colander.Invalid` exception is raised (:attr:`colander.null` + is not returned, as it is during serialization). + +Here's an example of a deserialization which uses a ``missing`` value +in the schema as a deserialization default value: + +.. code-block:: python + + import colander + + class Person(colander.MappingSchema): + name = colander.SchemaNode(colander.String()) + age = colander.SchemaNode(colander.Int(), missing=None) + + schema = Person() + deserialized = schema.deserialize({'name':'Fred', 'age':colander.null}) + +The value for ``deserialized`` above will be ``{'name':'Fred, +'age':None}``. + +Because the ``age`` schema node is provided a ``missing`` value of +``None``, if that schema is used to deserialize a mapping that has an +an ``age`` key of :attr:`colander.null`, the ``missing`` value of +``None`` is serialized into the appstruct output for ``age``. + +.. note:: Note that ``None`` can be used for the ``missing`` schema + node value as required, as in the above example. It's no different + than any other value used as ``missing``. + +The :attr:`colander.null` value is also the default, so it needn't be +specified in the cstruct. Therefore, the ``deserialized`` value of +the below is equivalent to the above's: + +.. code-block:: python + + import colander + + class Person(colander.MappingSchema): + name = colander.SchemaNode(colander.String()) + age = colander.SchemaNode(colander.Int(), missing=None) + + schema = Person() + deserialized = schema.deserialize({'name':'Fred'}) + +Deserialization Combinations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Within this table, the ``Value`` column represents the value passed to +the :meth:`colander.SchemaNode.deserialize` method of a particular +schema node, the ``Missing`` column represents the ``missing`` value +of that schema node, and the ``Result`` column is a description of the +result of invoking the :meth:`colander.SchemaNode.deserialize` method +of the schema node with the effective value. + +===================== ===================== =========================== +Value Missing Result +===================== ===================== =========================== +colander.null Invalid exception raised + Invalid exception raised +colander.null colander.null Invalid exception raised +colander.null value value deserialized + value value deserialized +value value deserialized +value colander.null value deserialized +value_a value_b value_a deserialized +===================== ===================== =========================== + +.. note:: ```` in the above table represents the circumstance + in which a key present in a :class:`colander.MappingSchema` is not + present in a mapping passed to its + :meth:`colander.SchemaNode.deserialize` method. In reality, + ```` means exactly the same thing as + :attr:`colander.null`, because the :class:`colander.Mapping` + type does the equivalent of ``mapping.get(keyname, + colander.null)`` to find a subvalue during deserialization. +