- SchemaNode deserialization now unconditionally calls the schema type's

``deserialize`` method to obtain an appstruct before attempting to
  validate.  Third party schema types should now return ``colander.null`` if
  passed a ``colander.null`` value or another logically "empty" value as a
  cstruct during ``deserialize``.

CHANGES.txt

@@ -19,6 +19,12 @@ Next release
 
 - Add SchemaNode.__contains__ to support "name in schema".
 
+- SchemaNode deserialization now unconditionally calls the schema type's
+  ``deserialize`` method to obtain an appstruct before attempting to
+  validate.  Third party schema types should now return ``colander.null`` if
+  passed a ``colander.null`` value or another logically "empty" value as a
+  cstruct during ``deserialize``.
+
 0.9.1 (2010-12-02)
 ------------------
 

colander/__init__.py

@@ -453,6 +453,8 @@ class Mapping(SchemaType):
         return self._impl(node, appstruct, callback)
 
     def deserialize(self, node, cstruct):
+        if cstruct is null:
+            return null
 
         def callback(subnode, subcstruct):
             return subnode.deserialize(subcstruct)
@@ -543,6 +545,9 @@ class Tuple(Positional, SchemaType):
         return self._impl(node, appstruct, callback)
 
     def deserialize(self, node, cstruct):
+        if cstruct is null:
+            return null
+        
         def callback(subnode, subval):
             return subnode.deserialize(subval)
 
@@ -671,6 +676,9 @@ class Sequence(Positional, SchemaType):
         respect the default ``accept_scalar`` value attached to this
         instance via its constructor.
         """
+        if cstruct is null:
+            return null
+        
         def callback(subnode, subcstruct):
             return subnode.deserialize(subcstruct)
 
@@ -698,7 +706,7 @@ Seq = Sequence
 class String(SchemaType):
     """ A type representing a Unicode string.
 
-    This type constructor accepts one argument:
+    This type constructor accepts two arguments:
 
     ``encoding``
        Represents the encoding which should be applied to value
@@ -751,6 +759,12 @@ class String(SchemaType):
        encoding.  If this is not true, an :exc:`colander.Invalid`
        error will result.
 
+    ``empty``
+
+       When an empty value is deserialized, empty represents the value passed
+       back to the caller of ``deserialize`` or ``validate``.  By default,
+       this is ``colander.null``.
+
     The subnodes of the :class:`colander.SchemaNode` that wraps
     this type are ignored.
     """
@@ -780,6 +794,9 @@ class String(SchemaType):
                             mapping={'val':appstruct, 'err':e})
                           )
     def deserialize(self, node, cstruct):
+        if not cstruct:
+            return null
+        
         try:
             result = cstruct
             if not isinstance(result, unicode):
@@ -813,9 +830,9 @@ class Number(SchemaType):
                             mapping={'val':appstruct}),
                           )
     def deserialize(self, node, cstruct):
-        if not cstruct:
+        if cstruct != 0 and not cstruct:
-            raise Invalid(node, _('Required'))
+            return null
-
+        
         try:
             return self.num(cstruct)
         except Exception:
@@ -889,6 +906,9 @@ class Boolean(SchemaType):
         return appstruct and 'true' or 'false'
 
     def deserialize(self, node, cstruct):
+        if cstruct is null:
+            return null
+        
         try:
             result = str(cstruct)
         except:
@@ -1014,6 +1034,9 @@ class GlobalObject(SchemaType):
                             mapping={'val':appstruct})
                           )
     def deserialize(self, node, cstruct):
+        if not cstruct:
+            return null
+        
         if not isinstance(cstruct, basestring):
             raise Invalid(node,
                           _('"${val}" is not a string',
@@ -1097,6 +1120,9 @@ class DateTime(SchemaType):
         return appstruct.isoformat()
 
     def deserialize(self, node, cstruct):
+        if not cstruct:
+            return null
+        
         try:
             result = iso8601.parse_date(cstruct)
         except (iso8601.ParseError, TypeError), e:
@@ -1167,6 +1193,8 @@ class Date(SchemaType):
         return appstruct.isoformat()
 
     def deserialize(self, node, cstruct):
+        if not cstruct:
+            return null
         try:
             result = iso8601.parse_date(cstruct)
             result = result.date()
@@ -1314,25 +1342,31 @@ class SchemaNode(object):
         this node using the fstruct passed. """
 
     def deserialize(self, cstruct=null):
-        """ Deserialize and validate the :term:`cstruct` into an
+        """ Deserialize the :term:`cstruct` into an :term:`appstruct` based
-        :term:`appstruct` based on the schema, and return the
+        on the schema, and return the deserialized, then validate the
-        deserialized, validated appstruct.  If the cstruct cannot be
+        resulting appstruct.  The ``cstruct`` value is deserialized into an
-        validated, a :exc:`colander.Invalid` exception will be raised.
+        ``appstruct`` unconditionally.
 
-        If ``cstruct`` is :attr:`colander.null`, do something special:
+        If ``appstruct`` returned by type deserialization is the value
+        :attr:`colander.null`, do something special before attempting
+        validation:
 
-        - If the ``missing`` attribute of this node has been set
+        - If the ``missing`` attribute of this node has been set explicitly,
-          explicitly, return its value.  No deserialization or
+          return its value.  No validation of this value is performed; it is
-          validation of this value is performed; it is simply
+          simply returned.
-          returned.
 
         - If the ``missing`` attribute of this node has not been set
           explicitly, raise a :exc:`colander.Invalid` exception error.
 
+        If the appstruct is not ``colander.null`` and cannot be validated , a
+        :exc:`colander.Invalid` exception will be raised.
+
         If a ``cstruct`` argument is not explicitly provided, it
         defaults to :attr:`colander.null`.
         """
-        if cstruct is null:
+        appstruct = self.typ.deserialize(self, cstruct)
+
+        if appstruct is null:
             appstruct = self.missing
             if appstruct is required:
                 raise Invalid(self, _('Required'))
@@ -1341,7 +1375,6 @@ class SchemaNode(object):
             # We never deserialize or validate the missing value
             return appstruct
 
-        appstruct = self.typ.deserialize(self, cstruct)
         if self.validator is not None:
             if not isinstance(self.validator, deferred): # unbound
                 self.validator(self, appstruct)

colander/tests.py

@@ -354,6 +354,13 @@ class TestMapping(unittest.TestCase):
         self.failUnless(
             e.msg.interpolate().startswith('"None" is not a mapping type'))
 
+    def test_deserialize_null(self):
+        import colander
+        node = DummySchemaNode(None)
+        typ = self._makeOne()
+        result = typ.deserialize(node, colander.null)
+        self.assertEqual(result, colander.null)
+
     def test_deserialize_no_subnodes(self):
         node = DummySchemaNode(None)
         typ = self._makeOne()
@@ -483,6 +490,13 @@ class TestTuple(unittest.TestCase):
         result = typ.deserialize(node, ())
         self.assertEqual(result, ())
 
+    def test_deserialize_null(self):
+        import colander
+        node = DummySchemaNode(None)
+        typ = self._makeOne()
+        result = typ.deserialize(node, colander.null)
+        self.assertEqual(result, colander.null)
+
     def test_deserialize_ok(self):
         node = DummySchemaNode(None)
         node.children = [DummySchemaNode(None, name='a')]
@@ -620,6 +634,12 @@ class TestSequence(unittest.TestCase):
         result = typ.deserialize(node, ())
         self.assertEqual(result, [])
 
+    def test_deserialize_no_null(self):
+        import colander
+        typ = self._makeOne()
+        result = typ.deserialize(None, colander.null)
+        self.assertEqual(result, colander.null)
+
     def test_deserialize_ok(self):
         node = DummySchemaNode(None)
         node.children = [DummySchemaNode(None, name='a')]
@@ -711,10 +731,11 @@ class TestString(unittest.TestCase):
         self.assertEqual(Str, String)
 
     def test_deserialize_emptystring(self):
+        from colander import null
         node = DummySchemaNode(None)
         typ = self._makeOne(None)
         result = typ.deserialize(node, '')
-        self.assertEqual(result, '')
+        self.assertEqual(result, null)
 
     def test_deserialize_uncooperative(self):
         val = Uncooperative()
@@ -822,12 +843,13 @@ class TestInteger(unittest.TestCase):
         result = typ.serialize(node, val)
         self.assertEqual(result, colander.null)
 
-    def test_serialize_emptystring_required(self):
+    def test_serialize_emptystring(self):
+        import colander
         val = ''
         node = DummySchemaNode(None)
         typ = self._makeOne()
-        e = invalid_exc(typ.deserialize, node, val)
+        result = typ.deserialize(node, val)
-        self.assertEqual(e.msg, 'Required')
+        self.assertEqual(result, colander.null)
 
     def test_deserialize_fails(self):
         val = 'P'
@@ -870,12 +892,13 @@ class TestFloat(unittest.TestCase):
         result = typ.serialize(node, val)
         self.assertEqual(result, colander.null)
 
-    def test_serialize_emptystring_required(self):
+    def test_serialize_emptystring(self):
+        import colander
         val = ''
         node = DummySchemaNode(None)
         typ = self._makeOne()
-        e = invalid_exc(typ.deserialize, node, val)
+        result = typ.deserialize(node, val)
-        self.assertEqual(e.msg, 'Required')
+        self.assertEqual(result, colander.null)
 
     def test_deserialize_fails(self):
         val = 'P'
@@ -918,12 +941,13 @@ class TestDecimal(unittest.TestCase):
         result = typ.serialize(node, val)
         self.assertEqual(result, colander.null)
 
-    def test_serialize_emptystring_required(self):
+    def test_serialize_emptystring(self):
+        import colander
         val = ''
         node = DummySchemaNode(None)
         typ = self._makeOne()
-        e = invalid_exc(typ.deserialize, node, val)
+        result = typ.deserialize(node, val)
-        self.assertEqual(e.msg, 'Required')
+        self.assertEqual(result, colander.null)
 
     def test_deserialize_fails(self):
         val = 'P'
@@ -987,6 +1011,13 @@ class TestBoolean(unittest.TestCase):
         e = invalid_exc(typ.deserialize, node, Uncooperative())
         self.failUnless(e.msg.endswith('not a string'))
 
+    def test_deserialize_null(self):
+        import colander
+        typ = self._makeOne()
+        node = DummySchemaNode(None)
+        result = typ.deserialize(node, colander.null)
+        self.assertEqual(result, colander.null)
+
     def test_serialize(self):
         typ = self._makeOne()
         node = DummySchemaNode(None)
@@ -1110,11 +1141,25 @@ class TestGlobalObject(unittest.TestCase):
         self.assertRaises(ImportError, typ._pkg_resources_style, None,
                           ':notexisting')
 
-    def test_deserialize_not_a_string(self):
+    def test_deserialize_None(self):
+        import colander
         typ = self._makeOne()
         node = DummySchemaNode(None)
-        e = invalid_exc(typ.deserialize, node, None)
+        result = typ.deserialize(node, None)
-        self.assertEqual(e.msg.interpolate(), '"None" is not a string')
+        self.assertEqual(result, colander.null)
+
+    def test_deserialize_null(self):
+        import colander
+        typ = self._makeOne()
+        node = DummySchemaNode(None)
+        result = typ.deserialize(node, colander.null)
+        self.assertEqual(result, colander.null)
+
+    def test_deserialize_notastring(self):
+        import colander
+        typ = self._makeOne()
+        node = DummySchemaNode(None)
+        self.assertRaises(colander.Invalid, typ.deserialize, node, True)
 
     def test_deserialize_using_pkgresources_style(self):
         typ = self._makeOne()
@@ -1243,6 +1288,20 @@ class TestDateTime(unittest.TestCase):
         e = invalid_exc(typ.deserialize, node, 'garbage')
         self.failUnless('Invalid' in e.msg)
 
+    def test_deserialize_null(self):
+        import colander
+        node = DummySchemaNode(None)
+        typ = self._makeOne()
+        result = typ.deserialize(node, colander.null)
+        self.assertEqual(result, colander.null)
+
+    def test_deserialize_empty(self):
+        import colander
+        node = DummySchemaNode(None)
+        typ = self._makeOne()
+        result = typ.deserialize(node, '')
+        self.assertEqual(result, colander.null)
+
     def test_deserialize_success(self):
         import iso8601
         typ = self._makeOne()
@@ -1309,6 +1368,20 @@ class TestDate(unittest.TestCase):
         e = invalid_exc(typ.deserialize, node, '10-10-10-10')
         self.failUnless('Invalid' in e.msg)
 
+    def test_deserialize_null(self):
+        import colander
+        node = DummySchemaNode(None)
+        typ = self._makeOne()
+        result = typ.deserialize(node, colander.null)
+        self.assertEqual(result, colander.null)
+
+    def test_deserialize_empty(self):
+        import colander
+        node = DummySchemaNode(None)
+        typ = self._makeOne()
+        result = typ.deserialize(node, '')
+        self.assertEqual(result, colander.null)
+
     def test_deserialize_success_date(self):
         typ = self._makeOne()
         date = self._today()