add flatten API to all builtin types

CHANGES.txt

@@ -10,6 +10,18 @@ Changes
 
 - Added Spanish locale:  thanks to Douglas Cerna for the translations!
 
+- If you use a schema with deferred ``validator``, ``missing`` or
+  ``default`` attributes, but you use it to perform serialization and
+  deserialization without calling its ``bind`` method:
+
+  - If ``validator`` is deferred, no validation will be performed.
+
+  - If ``missing`` is deferred, the field will be considered *required*.
+
+  - If ``default`` is deferred, the serialization default will be
+    assumed to be ``colander.null``.
+
+
 0.8  (2010/09/08)
 -----------------
 

colander/__init__.py

@@ -329,7 +329,15 @@ class OneOf(object):
                     mapping={'val':value, 'choices':choices})
             raise Invalid(node, err)
 
-class Mapping(object):
+class SchemaType(object):
+    """ Base class for all schema types """
+    def flatten(self, node, appstruct, prefix=''):
+        result = {}
+        selfname = '%s%s' % (prefix, node.name)
+        result[selfname] = appstruct
+        return result
+
+class Mapping(SchemaType):
     """ A type which represents a mapping of names to nodes.
 
     The subnodes of the :class:`colander.SchemaNode` that wraps
@@ -447,6 +455,19 @@ class Mapping(object):
 
         return self._impl(node, cstruct, callback)
 
+    def flatten(self, node, appstruct, prefix=''):
+        result = {}
+        selfname = '%s%s' % (prefix, node.name)
+        selfprefix = selfname + '.'
+        result[selfname] = appstruct
+
+        for num, subnode in enumerate(node.children):
+            name = subnode.name
+            substruct = appstruct.get(name, null)
+            result.update(subnode.typ.flatten(subnode, substruct,
+                                              prefix=selfprefix))
+        return result
+        
 class Positional(object):
     """
     Marker abstract base class meaning 'this type has children which
@@ -455,7 +476,7 @@ class Positional(object):
     creating a dictionary representation of an error tree.
     """
 
-class Tuple(Positional):
+class Tuple(Positional, SchemaType):
     """ A type which represents a fixed-length sequence of nodes.
 
     The subnodes of the :class:`colander.SchemaNode` that wraps
@@ -523,7 +544,19 @@ class Tuple(Positional):
 
         return self._impl(node, cstruct, callback)
 
-class Sequence(Positional):
+    def flatten(self, node, appstruct, prefix=''):
+        result = {}
+        selfname = '%s%s' % (prefix, node.name)
+        selfprefix = selfname + '.'
+        result[selfname] = appstruct
+
+        for num, subnode in enumerate(node.children):
+            substruct = appstruct[num]
+            result.update(subnode.typ.flatten(subnode, substruct,
+                                              prefix=selfprefix))
+        return result
+
+class Sequence(Positional, SchemaType):
     """
     A type which represents a variable-length sequence of nodes,
     all of which must be of the same type.
@@ -639,9 +672,26 @@ class Sequence(Positional):
 
         return self._impl(node, cstruct, callback, accept_scalar)
 
+    def flatten(self, node, appstruct, prefix=''):
+        result = {}
+        selfname = '%s%s' % (prefix, node.name)
+        selfprefix = selfname + '.'
+        result[selfname] = appstruct
+
+        childnode = node.children[0]
+
+        for num, subval in enumerate(appstruct):
+            subname = '%s%s' % (selfprefix, num)
+            result[subname] = subval
+            subprefix = subname + '.'
+            result.update(childnode.typ.flatten(childnode, subval,
+                                                prefix=subprefix))
+
+        return result
+
 Seq = Sequence
 
-class String(object):
+class String(SchemaType):
     """ A type representing a Unicode string.
 
     This type constructor accepts one argument:
@@ -740,10 +790,9 @@ class String(object):
 
         return result
 
-
 Str = String
 
-class Number(object):
+class Number(SchemaType):
     """ Abstract base class for float, int, decimal """
 
     num = None
@@ -771,7 +820,6 @@ class Number(object):
                             mapping={'val':cstruct})
                           )
 
-
 class Integer(Number):
     """ A type representing an integer.
 
@@ -812,7 +860,7 @@ class Decimal(Number):
     def num(self, val):
         return decimal.Decimal(str(val))
 
-class Boolean(object):
+class Boolean(SchemaType):
     """ A type representing a boolean object.
 
     During deserialization, a value in the set (``false``, ``0``) will
@@ -852,7 +900,7 @@ class Boolean(object):
 
 Bool = Boolean
 
-class GlobalObject(object):
+class GlobalObject(SchemaType):
     """ A type representing an importable Python object.  This type
     serializes 'global' Python objects (objects which can be imported)
     to dotted Python names.
@@ -976,8 +1024,7 @@ class GlobalObject(object):
                           _('The dotted name "${name}" cannot be imported',
                             mapping={'name':cstruct}))
 
-
+class DateTime(SchemaType):
-class DateTime(object):
     """ A type representing a Python ``datetime.datetime`` object.
 
     This type serializes python ``datetime.datetime`` objects to a
@@ -1058,7 +1105,7 @@ class DateTime(object):
                                       mapping={'val':cstruct, 'err':e}))
         return result
 
-class Date(object):
+class Date(SchemaType):
     """ A type representing a Python ``datetime.date`` object.
 
     This type serializes python ``datetime.date`` objects to a
@@ -1220,6 +1267,8 @@ class SchemaNode(object):
         A return value of ``True`` implies that a ``missing`` value
         wasn't specified for this node.  A return value of ``False``
         implies that a ``missing`` value was specified for this node."""
+        if isinstance(self.missing, deferred):  # unbound schema with deferreds
+            return True
         return self.missing is _marker
 
     def serialize(self, appstruct=null):
@@ -1236,9 +1285,25 @@ class SchemaNode(object):
         """
         if appstruct is null:
             appstruct = self.default
+        if isinstance(appstruct, deferred): # unbound schema with deferreds
+            appstruct = null
         cstruct = self.typ.serialize(self, appstruct)
         return cstruct
 
+    def flatten(self, appstruct):
+        """ Create an fstruct from the appstruct based on the schema
+        represented by this node and return the fstruct.  An fstruct
+        is a flattened representation of an appstruct.  An fstruct is
+        a dictionary; its keys are dotted names.  Each dotted name
+        represents a location in the schema.  The values of an fstruct
+        dictionary are appstruct subvalues."""
+        flat = self.typ.flatten(self, appstruct)
+        return flat
+
+    def unflatten(self, fstruct):
+        """ Create an appstruct based on the schema represented by
+        this node using the fstruct passed. """
+
     def deserialize(self, cstruct=null):
         """ Deserialize and validate the :term:`cstruct` into an
         :term:`appstruct` based on the schema, and return the
@@ -1262,12 +1327,15 @@ class SchemaNode(object):
             appstruct = self.missing
             if appstruct is _marker:
                 raise Invalid(self, _('Required'))
+            if isinstance(appstruct, deferred): # unbound schema with deferreds
+                raise Invalid(self, _('Required'))
             # We never deserialize or validate the missing value
             return appstruct
 
         appstruct = self.typ.deserialize(self, cstruct)
         if self.validator is not None:
-            self.validator(self, appstruct)
+            if not isinstance(self.validator, deferred): # unbound
+                self.validator(self, appstruct)
         return appstruct
 
     def add(self, node):

colander/tests.py

@@ -319,6 +319,17 @@ class TestOneOf(unittest.TestCase):
         e = invalid_exc(validator, None, None)
         self.assertEqual(e.msg.interpolate(), '"None" is not one of 1, 2')
 
+class TestSchemaType(unittest.TestCase):
+    def _makeOne(self, *arg, **kw):
+        from colander import SchemaType
+        return SchemaType(*arg, **kw)
+
+    def test_flatten(self):
+        node = DummySchemaNode(None, name='node')
+        typ = self._makeOne()
+        result = typ.flatten(node, 'appstruct')
+        self.assertEqual(result, {'node':'appstruct'})
+
 
 class TestMapping(unittest.TestCase):
     def _makeOne(self, *arg, **kw):
@@ -439,6 +450,19 @@ class TestMapping(unittest.TestCase):
         result = typ.serialize(node, null)
         self.assertEqual(result, {'a':null})
 
+    def test_flatten(self):
+        node = DummySchemaNode(None, name='node')
+        int1 = DummyType()
+        int2 = DummyType()
+        node.children = [
+            DummySchemaNode(int1, name='a'),
+            DummySchemaNode(int2, name='b'),
+            ]
+        typ = self._makeOne()
+        result = typ.flatten(node, {'a':1, 'b':2})
+        self.assertEqual(result,
+                         {'node': {'a': 1, 'b': 2}, 'node.appstruct': 2})
+
 class TestTuple(unittest.TestCase):
     def _makeOne(self):
         from colander import Tuple
@@ -550,6 +574,18 @@ class TestTuple(unittest.TestCase):
         self.assertEqual(e.msg, None)
         self.assertEqual(len(e.children), 2)
 
+    def test_flatten(self):
+        node = DummySchemaNode(None, name='node')
+        int1 = DummyType()
+        int2 = DummyType()
+        node.children = [
+            DummySchemaNode(int1, name='a'),
+            DummySchemaNode(int2, name='b'),
+            ]
+        typ = self._makeOne()
+        result = typ.flatten(node, (1, 2))
+        self.assertEqual(result, {'node': (1, 2), 'node.appstruct': 2})
+
 class TestSequence(unittest.TestCase):
     def _makeOne(self, **kw):
         from colander import Sequence
@@ -646,6 +682,24 @@ class TestSequence(unittest.TestCase):
         self.assertEqual(e.msg, None)
         self.assertEqual(len(e.children), 2)
 
+    def test_flatten(self):
+        node = DummySchemaNode(None, name='node')
+        int1 = DummyType()
+        int2 = DummyType()
+        node.children = [
+            DummySchemaNode(int1, name='a'),
+            DummySchemaNode(int2, name='b'),
+            ]
+        typ = self._makeOne()
+        result = typ.flatten(node, [1, 2])
+        self.assertEqual(
+            result,
+            {'node': [1, 2],
+             'node.0': 1,
+             'node.0.appstruct': 1,
+             'node.1.appstruct': 2,
+             'node.1': 2})
+
 class TestString(unittest.TestCase):
     def _makeOne(self, encoding=None):
         from colander import String
@@ -1322,6 +1376,11 @@ class TestSchemaNode(unittest.TestCase):
         node = self._makeOne(None, missing=1)
         self.assertEqual(node.required, False)
 
+    def test_required_deferred(self):
+        from colander import deferred
+        node = self._makeOne(None, missing=deferred('123'))
+        self.assertEqual(node.required, True)
+
     def test_deserialize_no_validator(self):
         typ = DummyType()
         node = self._makeOne(typ)
@@ -1362,6 +1421,15 @@ class TestSchemaNode(unittest.TestCase):
         node.missing = null
         self.assertEqual(node.deserialize(null), null)
 
+    def test_deserialize_appstruct_deferred(self):
+        from colander import null
+        from colander import deferred
+        from colander import Invalid
+        typ = DummyType()
+        node = self._makeOne(typ)
+        node.missing = deferred('123')
+        self.assertRaises(Invalid, node.deserialize, null)
+
     def test_serialize(self):
         typ = DummyType()
         node = self._makeOne(typ)
@@ -1389,6 +1457,14 @@ class TestSchemaNode(unittest.TestCase):
         node.default = 'abc'
         self.assertEqual(node.serialize(), 'abc')
 
+    def test_serialize_default_deferred(self):
+        from colander import deferred
+        from colander import null
+        typ = DummyType()
+        node = self._makeOne(typ)
+        node.default = deferred('abc')
+        self.assertEqual(node.serialize(), null)
+
     def test_add(self):
         node = self._makeOne(None)
         node.add(1)
@@ -1595,20 +1671,73 @@ class TestFunctional(object):
         self.assertEqual(result['seq2'],
                          [{'key':1, 'key2':2}, {'key':3, 'key2':4}])
         self.assertEqual(result['tup'], (1, 's'))
+
+    def test_flatten_ok(self):
+        import colander
+        appstruct = {
+            'int':10,
+            'ob':colander.tests,
+            'seq':[(1, 's'),(2, 's'), (3, 's'), (4, 's')],
+            'seq2':[{'key':1, 'key2':2}, {'key':3, 'key2':4}],
+            'tup':(1, 's'),
+            }
+        schema = self._makeSchema()
+        result = schema.flatten(appstruct)
+
+        expected = {
+            'schema.seq.2.tup.tupstring': 's',
+            'schema.seq2.0.mapping.key2': 2,
+            'schema.seq.0': (1, 's'),
+            'schema.seq.1': (2, 's'),
+            'schema.seq.2': (3, 's'),
+            'schema.seq.3': (4, 's'),
+            'schema.seq': [(1, 's'), (2, 's'), (3, 's'), (4, 's')],
+            'schema.ob': colander.tests,
+            'schema.seq2.1.mapping.key2': 4,
+            'schema.seq.0.tup': (1, 's'),
+            'schema.seq.1.tup': (2, 's'),
+            'schema.seq2.0.mapping': {'key2': 2, 'key': 1},
+            'schema.seq2.1.mapping': {'key2': 4, 'key': 3},
+            'schema.seq.1.tup.tupstring': 's',
+            'schema.seq2.0.mapping.key': 1,
+            'schema.seq.1.tup.tupint': 2,
+            'schema.tup': (1, 's'),
+            'schema.seq.3.tup': (4, 's'),
+            'schema.seq.0.tup.tupstring': 's',
+            'schema.seq.2.tup': (3, 's'),
+            'schema.seq.3.tup.tupstring': 's',
+            'schema.seq.3.tup.tupint': 4,
+            'schema.seq2.1.mapping.key': 3,
+            'schema.int': 10,
+            'schema.seq2.0': {'key2': 2, 'key': 1},
+            'schema.seq.0.tup.tupint': 1,
+            'schema.tup.tupint': 1,
+            'schema.tup.tupstring': 's',
+            'schema.seq.2.tup.tupint': 3,
+            'schema.seq2': [{'key2': 2, 'key': 1}, {'key2': 4, 'key': 3}],
+            'schema.seq2.1': {'key2': 4, 'key': 3},
+            'schema': {'int': 10,
+                       'seq2': [{'key2': 2, 'key': 1}, {'key2': 4, 'key': 3}],
+                       'tup': (1, 's'),
+                       'ob':colander.tests,
+                       'seq': [(1, 's'), (2, 's'), (3, 's'), (4, 's')]}}
+
+        for k, v in result.items():
+            self.assertEqual(v, expected[k])
         
     def test_invalid_asdict(self):
         expected = {
-            'int': '20 is greater than maximum value 10',
+            'schema.int': '20 is greater than maximum value 10',
-            'ob': 'The dotted name "no.way.this.exists" cannot be imported',
+            'schema.ob': 'The dotted name "no.way.this.exists" cannot be imported',
-            'seq.0.0': '"q" is not a number',
+            'schema.seq.0.0': '"q" is not a number',
-            'seq.1.0': '"w" is not a number',
+            'schema.seq.1.0': '"w" is not a number',
-            'seq.2.0': '"e" is not a number',
+            'schema.seq.2.0': '"e" is not a number',
-            'seq.3.0': '"r" is not a number',
+            'schema.seq.3.0': '"r" is not a number',
-            'seq2.0.key': '"t" is not a number',
+            'schema.seq2.0.key': '"t" is not a number',
-            'seq2.0.key2': '"y" is not a number',
+            'schema.seq2.0.key2': '"y" is not a number',
-            'seq2.1.key': '"u" is not a number',
+            'schema.seq2.1.key': '"u" is not a number',
-            'seq2.1.key2': '"i" is not a number',
+            'schema.seq2.1.key2': '"i" is not a number',
-            'tup.0': '"s" is not a number'}
+            'schema.tup.0': '"s" is not a number'}
         data = {
             'int':'20',
             'ob':'no.way.this.exists',
@@ -1679,7 +1808,8 @@ class TestImperative(unittest.TestCase, TestFunctional):
             ob,
             tup,
             seq,
-            seq2)
+            seq2,
+            name='schema')
 
         return schema
 
@@ -1699,7 +1829,7 @@ class TestDeclarative(unittest.TestCase, TestFunctional):
             key2 = colander.SchemaNode(colander.Int())
 
         class SequenceOne(colander.SequenceSchema):
-            tuple = TupleSchema()
+            tup = TupleSchema()
 
         class SequenceTwo(colander.SequenceSchema):
             mapping = MappingSchema()
@@ -1712,7 +1842,7 @@ class TestDeclarative(unittest.TestCase, TestFunctional):
             tup = TupleSchema()
             seq2 = SequenceTwo()
 
-        schema = MainSchema()
+        schema = MainSchema(name='schema')
         return schema
 
 class Test_null(unittest.TestCase):
@@ -1770,3 +1900,7 @@ class DummyType(object):
     def deserialize(self, node, value):
         return value
 
+    def flatten(self, node, appstruct, prefix=''):
+        key = prefix + 'appstruct'
+        return {key:appstruct}
+

docs/binding.rst

@@ -237,6 +237,20 @@ will be the set of keywords passed to the ``bind`` method.  It usually
 operates on the ``node`` it is passed using the API methods described
 in :class:`SchemaNode`.
 
+Unbound Schemas With Deferreds
+------------------------------
+
+If you use a schema with deferred ``validator``, ``missing`` or
+``default`` attributes, but you use it to perform serialization and
+deserialization without calling its ``bind`` method:
+
+- If ``validator`` is deferred, no validation will be performed.
+
+- If ``missing`` is deferred, the field will be considered *required*.
+
+- If ``default`` is deferred, the serialization default will be
+  assumed to be ``colander.null``.
+
 See Also
 --------