diff --git a/CHANGES.txt b/CHANGES.txt index 4679894..7b82b47 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,16 @@ Changes ======= +Unreleased +---------- + +- ``flatten`` now only includes leaf nodes in the flattened dict. + +- ``flatten`` does not include a path element for the name of the type node + for sequences. + +- ``unflatten`` is implemented. + 0.9.3 (2011-06-23) ------------------ diff --git a/colander/__init__.py b/colander/__init__.py index 4d21dae..a03980f 100644 --- a/colander/__init__.py +++ b/colander/__init__.py @@ -336,12 +336,20 @@ class OneOf(object): class SchemaType(object): """ Base class for all schema types """ - def flatten(self, node, appstruct, prefix=''): + def flatten(self, node, appstruct, prefix='', listitem=False): result = {} - selfname = '%s%s' % (prefix, node.name) + if listitem: + selfname = prefix + else: + selfname = '%s%s' % (prefix, node.name) result[selfname] = appstruct return result + def unflatten(self, node, paths, fstruct): + name = node.name + assert paths == [name], "paths should be [name] for leaf nodes." + return fstruct[name] + class Mapping(SchemaType): """ A type which represents a mapping of names to nodes. @@ -462,11 +470,12 @@ class Mapping(SchemaType): return self._impl(node, cstruct, callback) - def flatten(self, node, appstruct, prefix=''): + def flatten(self, node, appstruct, prefix='', listitem=False): result = {} - selfname = '%s%s' % (prefix, node.name) - selfprefix = selfname + '.' - result[selfname] = appstruct + if listitem: + selfprefix = prefix + else: + selfprefix = '%s%s.' % (prefix, node.name) for subnode in node.children: name = subnode.name @@ -475,6 +484,10 @@ class Mapping(SchemaType): prefix=selfprefix)) return result + def unflatten(self, node, paths, fstruct): + return _unflatten_mapping(node, paths, fstruct) + + class Positional(object): """ Marker abstract base class meaning 'this type has children which @@ -554,11 +567,12 @@ class Tuple(Positional, SchemaType): return self._impl(node, cstruct, callback) - def flatten(self, node, appstruct, prefix=''): + def flatten(self, node, appstruct, prefix='', listitem=False): result = {} - selfname = '%s%s' % (prefix, node.name) - selfprefix = selfname + '.' - result[selfname] = appstruct + if listitem: + selfprefix = prefix + else: + selfprefix = '%s%s.' % (prefix, node.name) for num, subnode in enumerate(node.children): substruct = appstruct[num] @@ -566,6 +580,13 @@ class Tuple(Positional, SchemaType): prefix=selfprefix)) return result + def unflatten(self, node, paths, fstruct): + mapstruct = _unflatten_mapping(node, paths, fstruct) + appstruct = [] + for subnode in node.children: + appstruct.append(mapstruct[subnode.name]) + return tuple(appstruct) + class Sequence(Positional, SchemaType): """ A type which represents a variable-length sequence of nodes, @@ -685,23 +706,38 @@ class Sequence(Positional, SchemaType): return self._impl(node, cstruct, callback, accept_scalar) - def flatten(self, node, appstruct, prefix=''): + def flatten(self, node, appstruct, prefix='', listitem=False): result = {} - selfname = '%s%s' % (prefix, node.name) - selfprefix = selfname + '.' - result[selfname] = appstruct + if listitem: + selfprefix = prefix + else: + selfprefix = '%s%s.' % (prefix, node.name) childnode = node.children[0] for num, subval in enumerate(appstruct): subname = '%s%s' % (selfprefix, num) - result[subname] = subval subprefix = subname + '.' - result.update(childnode.typ.flatten(childnode, subval, - prefix=subprefix)) + result.update(childnode.typ.flatten( + childnode, subval, prefix=subprefix, listitem=True)) return result + def unflatten(self, node, paths, fstruct): + only_child = node.children[0] + child_name = only_child.name + def get_child(name): + return only_child + def rewrite_subpath(subpath): + if '.' in subpath: + suffix = subpath.split('.', 1)[1] + return '%s.%s' % (child_name, suffix) + return child_name + mapstruct = _unflatten_mapping(node, paths, fstruct, + get_child, rewrite_subpath) + return [mapstruct[str(index)] for index in xrange(len(mapstruct))] + + Seq = Sequence class String(SchemaType): @@ -1422,6 +1458,8 @@ class SchemaNode(object): def unflatten(self, fstruct): """ Create an appstruct based on the schema represented by this node using the fstruct passed. """ + paths = sorted(fstruct.keys()) + return self.typ.unflatten(self, paths, fstruct) def deserialize(self, cstruct=null): """ Deserialize the :term:`cstruct` into an :term:`appstruct` based @@ -1599,3 +1637,45 @@ class deferred(object): def __call__(self, node, kw): return self.wrapped(node, kw) +def _unflatten_mapping(node, paths, fstruct, + get_child=None, rewrite_subpath=None): + if get_child is None: + get_child = node.__getitem__ + if rewrite_subpath is None: + def rewrite_subpath(subpath): + return subpath + node_name = node.name + prefix = node_name + '.' + prefix_len = len(prefix) + appstruct = {} + subfstruct = {} + subpaths = [] + curname = None + for path in paths: + if path == node_name: + # flattened structs contain non-leaf nodes which are ignored + # during unflattening. + continue + assert path.startswith(prefix), "Bad node: %s" % path + subpath = path[prefix_len:] + if '.' in subpath: + name = subpath[:subpath.index('.')] + else: + name = subpath + if curname is None: + curname = name + elif name != curname: + subnode = get_child(curname) + appstruct[curname] = subnode.typ.unflatten( + subnode, subpaths, subfstruct) + subfstruct = {} + subpaths = [] + curname = name + subpath = rewrite_subpath(subpath) + subfstruct[subpath] = fstruct[path] + subpaths.append(subpath) + if curname is not None: + subnode = get_child(curname) + appstruct[curname] = subnode.typ.unflatten( + subnode, subpaths, subfstruct) + return appstruct diff --git a/colander/tests.py b/colander/tests.py index 0af2c02..81f96c5 100644 --- a/colander/tests.py +++ b/colander/tests.py @@ -62,7 +62,7 @@ class TestInvalid(unittest.TestCase): def test_paths(self): exc1 = self._makeOne(None, 'exc1') - exc2 = self._makeOne(None, 'exc2') + exc2 = self._makeOne(None, 'exc2') exc3 = self._makeOne(None, 'exc3') exc4 = self._makeOne(None, 'exc4') exc1.add(exc2) @@ -97,7 +97,7 @@ class TestInvalid(unittest.TestCase): node4 = DummySchemaNode(Positional(), 'node4') exc1 = self._makeOne(node1, 'exc1') exc1.pos = 1 - exc2 = self._makeOne(node2, 'exc2') + exc2 = self._makeOne(node2, 'exc2') exc3 = self._makeOne(node3, 'exc3') exc4 = self._makeOne(node4, 'exc4') exc1.add(exc2, 2) @@ -228,7 +228,7 @@ class TestRegex(unittest.TestCase): def _makeOne(self, pattern): from colander import Regex return Regex(pattern) - + def test_valid_regex(self): self.assertEqual(self._makeOne('a')(None, 'a'), None) self.assertEqual(self._makeOne('[0-9]+')(None, '1111'), None) @@ -246,7 +246,7 @@ class TestRegex(unittest.TestCase): regex = re.compile('[0-9]+') self.assertEqual(self._makeOne(regex)(None, '01'), None) self.assertRaises(Invalid, self._makeOne(regex), None, 't') - + class TestEmail(unittest.TestCase): def _makeOne(self): @@ -265,7 +265,7 @@ class TestEmail(unittest.TestCase): validator = self._makeOne() e = invalid_exc(validator, None, '') self.assertEqual(e.msg, 'Invalid email address') - + def test_invalid_emails(self): validator = self._makeOne() from colander import Invalid @@ -330,6 +330,17 @@ class TestSchemaType(unittest.TestCase): result = typ.flatten(node, 'appstruct') self.assertEqual(result, {'node':'appstruct'}) + def test_flatten_listitem(self): + node = DummySchemaNode(None, name='node') + typ = self._makeOne() + result = typ.flatten(node, 'appstruct', listitem=True) + self.assertEqual(result, {'':'appstruct'}) + + def test_unflatten(self): + node = DummySchemaNode(None, name='node') + typ = self._makeOne() + result = typ.unflatten(node, ['node'], {'node': 'appstruct'}) + self.assertEqual(result, 'appstruct') class TestMapping(unittest.TestCase): def _makeOne(self, *arg, **kw): @@ -467,8 +478,62 @@ class TestMapping(unittest.TestCase): ] typ = self._makeOne() result = typ.flatten(node, {'a':1, 'b':2}) - self.assertEqual(result, - {'node': {'a': 1, 'b': 2}, 'node.appstruct': 2}) + self.assertEqual(result, {'node.appstruct': 2}) + + def test_flatten_listitem(self): + node = DummySchemaNode(None, name='node') + int1 = DummyType() + int2 = DummyType() + node.children = [ + DummySchemaNode(int1, name='a'), + DummySchemaNode(int2, name='b'), + ] + typ = self._makeOne() + result = typ.flatten(node, {'a':1, 'b':2}, listitem=True) + self.assertEqual(result, {'appstruct': 2}) + + def test_unflatten(self): + node = DummySchemaNode(None, name='node') + int1 = DummyType() + int2 = DummyType() + node.children = [ + DummySchemaNode(int1, name='a'), + DummySchemaNode(int2, name='b'), + ] + typ = self._makeOne() + result = typ.unflatten(node, + ['node', 'node.a', 'node.b'], + {'node': {'a':1, 'b':2}, 'node.a':1, 'node.b':2}) + self.assertEqual(result, {'a': 1, 'b': 2}) + + def test_unflatten_nested(self): + node = DummySchemaNode(None, name='node') + inttype = DummyType() + one = DummySchemaNode(self._makeOne(), name='one') + one.children = [ + DummySchemaNode(inttype, name='a'), + DummySchemaNode(inttype, name='b'), + ] + two = DummySchemaNode(self._makeOne(), name='two') + two.children = [ + DummySchemaNode(inttype, name='c'), + DummySchemaNode(inttype, name='d'), + ] + node.children = [one, two] + typ = self._makeOne() + result = typ.unflatten( + node, ['node', 'node.one', 'node.one.a', 'node.one.b', + 'node.two', 'node.two.c', 'node.two.d'], + {'node': {'one': {'a': 1, 'b': 2}, 'two': {'c': 3, 'd': 4}}, + 'node.one': {'a': 1, 'b': 2}, + 'node.two': {'c': 3, 'd': 4}, + 'node.one.a': 1, + 'node.one.b': 2, + 'node.two.c': 3, + 'node.two.d': 4,}) + self.assertEqual(result, { + 'one': {'a': 1, 'b': 2}, 'two': {'c': 3, 'd': 4}}) + class TestTuple(unittest.TestCase): def _makeOne(self): @@ -598,7 +663,32 @@ class TestTuple(unittest.TestCase): ] typ = self._makeOne() result = typ.flatten(node, (1, 2)) - self.assertEqual(result, {'node': (1, 2), 'node.appstruct': 2}) + self.assertEqual(result, {'node.appstruct': 2}) + + def test_flatten_listitem(self): + node = DummySchemaNode(None, name='node') + int1 = DummyType() + int2 = DummyType() + node.children = [ + DummySchemaNode(int1, name='a'), + DummySchemaNode(int2, name='b'), + ] + typ = self._makeOne() + result = typ.flatten(node, (1, 2), listitem=True) + self.assertEqual(result, {'appstruct': 2}) + + def test_unflatten(self): + node = DummySchemaNode(None, name='node') + int1 = DummyType() + int2 = DummyType() + node.children = [ + DummySchemaNode(int1, name='a'), + DummySchemaNode(int2, name='b'), + ] + typ = self._makeOne() + result = typ.unflatten(node, ['node', 'node.a', 'node.b'], + {'node': (1, 2), 'node.a': 1, 'node.b': 2}) + self.assertEqual(result, (1, 2)) class TestSequence(unittest.TestCase): def _makeOne(self, **kw): @@ -712,13 +802,31 @@ class TestSequence(unittest.TestCase): ] typ = self._makeOne() result = typ.flatten(node, [1, 2]) - self.assertEqual( - result, - {'node': [1, 2], - 'node.0': 1, - 'node.0.appstruct': 1, - 'node.1.appstruct': 2, - 'node.1': 2}) + self.assertEqual(result, {'node.0': 1, 'node.1': 2}) + + def test_flatten_listitem(self): + node = DummySchemaNode(None, name='node') + int1 = DummyType() + int2 = DummyType() + node.children = [ + DummySchemaNode(int1, name='a'), + DummySchemaNode(int2, name='b'), + ] + typ = self._makeOne() + result = typ.flatten(node, [1, 2], listitem=True) + self.assertEqual(result, {'0': 1, '1': 2}) + + def test_unflatten(self): + node = DummySchemaNode(None, name='node') + node.children = [ + DummySchemaNode(DummyType(), name='foo'), + ] + typ = self._makeOne() + result = typ.unflatten(node, + ['node.0', 'node.1',], + {'node.0': 'a', 'node.1': 'b'}) + self.assertEqual(result, ['a', 'b']) + class TestString(unittest.TestCase): def _makeOne(self, encoding=None): @@ -824,7 +932,7 @@ class TestString(unittest.TestCase): typ = self._makeOne('utf-8') e = invalid_exc(typ.serialize, node, not_utf8) self.failUnless('cannot be serialized' in e.msg) - + class TestInteger(unittest.TestCase): def _makeOne(self): from colander import Integer @@ -1036,7 +1144,7 @@ class TestGlobalObject(unittest.TestCase): result = typ._zope_dottedname_style(None, 'colander.tests.TestGlobalObject') self.assertEqual(result, self.__class__) - + def test_zope_dottedname_style_irrresolveable_absolute(self): typ = self._makeOne() self.assertRaises(ImportError, typ._zope_dottedname_style, None, @@ -1105,7 +1213,7 @@ class TestGlobalObject(unittest.TestCase): result = typ._pkg_resources_style(None, 'colander.tests:TestGlobalObject') self.assertEqual(result, self.__class__) - + def test__pkg_resources_style_irrresolveable_absolute(self): typ = self._makeOne() self.assertRaises(ImportError, typ._pkg_resources_style, None, @@ -1128,7 +1236,7 @@ class TestGlobalObject(unittest.TestCase): typ = self._makeOne(package=colander.tests) result = typ._pkg_resources_style(None, '.') self.assertEqual(result, colander.tests) - + def test__pkg_resources_style_resolve_relative_nocurrentpackage(self): typ = self._makeOne() import colander @@ -1194,7 +1302,7 @@ class TestGlobalObject(unittest.TestCase): node = DummySchemaNode(None) result = typ.serialize(node, colander.tests) self.assertEqual(result, 'colander.tests') - + def test_serialize_fail(self): typ = self._makeOne() node = DummySchemaNode(None) @@ -1711,7 +1819,7 @@ class TestSchemaNode(unittest.TestCase): another = self._makeOne(None, name='another') node.add(another) self.assertEqual(node['another'], another) - + def test___getitem__failure(self): node = self._makeOne(None) self.assertRaises(KeyError, node.__getitem__, 'another') @@ -1722,7 +1830,7 @@ class TestSchemaNode(unittest.TestCase): node.add(another) del node['another'] self.assertEqual(node.children, []) - + def test___delitem__failure(self): node = self._makeOne(None) self.assertRaises(KeyError, node.__delitem__, 'another') @@ -1841,7 +1949,7 @@ class TestSchema(unittest.TestCase): self.assertEqual(node.default, 'abc') self.assertEqual(node.__class__, colander.SchemaNode) self.assertEqual(node.typ.__class__, colander.Mapping) - self.assertEqual(node.children[0].typ.__class__, colander.String) + self.assertEqual(node.children[0].typ.__class__, colander.String) self.assertEqual(node.children[0].title, 'Thing A') self.assertEqual(node.children[1].title, 'bar') @@ -1935,46 +2043,79 @@ class TestFunctional(object): result = schema.flatten(appstruct) expected = { - 'schema.seq.2.tup.tupstring': 's', - 'schema.seq2.0.mapping.key2': 2, - 'schema.seq.0': (1, 's'), - 'schema.seq.1': (2, 's'), - 'schema.seq.2': (3, 's'), - 'schema.seq.3': (4, 's'), - 'schema.seq': [(1, 's'), (2, 's'), (3, 's'), (4, 's')], + 'schema.seq.2.tupstring': 's', + 'schema.seq2.0.key2': 2, 'schema.ob': colander.tests, - 'schema.seq2.1.mapping.key2': 4, - 'schema.seq.0.tup': (1, 's'), - 'schema.seq.1.tup': (2, 's'), - 'schema.seq2.0.mapping': {'key2': 2, 'key': 1}, - 'schema.seq2.1.mapping': {'key2': 4, 'key': 3}, - 'schema.seq.1.tup.tupstring': 's', - 'schema.seq2.0.mapping.key': 1, - 'schema.seq.1.tup.tupint': 2, - 'schema.tup': (1, 's'), - 'schema.seq.3.tup': (4, 's'), - 'schema.seq.0.tup.tupstring': 's', - 'schema.seq.2.tup': (3, 's'), - 'schema.seq.3.tup.tupstring': 's', - 'schema.seq.3.tup.tupint': 4, - 'schema.seq2.1.mapping.key': 3, + 'schema.seq2.1.key2': 4, + 'schema.seq.1.tupstring': 's', + 'schema.seq2.0.key': 1, + 'schema.seq.1.tupint': 2, + 'schema.seq.0.tupstring': 's', + 'schema.seq.3.tupstring': 's', + 'schema.seq.3.tupint': 4, + 'schema.seq2.1.key': 3, 'schema.int': 10, - 'schema.seq2.0': {'key2': 2, 'key': 1}, - 'schema.seq.0.tup.tupint': 1, + 'schema.seq.0.tupint': 1, 'schema.tup.tupint': 1, 'schema.tup.tupstring': 's', - 'schema.seq.2.tup.tupint': 3, - 'schema.seq2': [{'key2': 2, 'key': 1}, {'key2': 4, 'key': 3}], - 'schema.seq2.1': {'key2': 4, 'key': 3}, - 'schema': {'int': 10, - 'seq2': [{'key2': 2, 'key': 1}, {'key2': 4, 'key': 3}], - 'tup': (1, 's'), - 'ob':colander.tests, - 'seq': [(1, 's'), (2, 's'), (3, 's'), (4, 's')]}} + 'schema.seq.2.tupint': 3, + } + for k, v in expected.items(): + self.assertEqual(result[k], v) for k, v in result.items(): - self.assertEqual(v, expected[k]) - + self.assertEqual(expected[k], v) + + def test_unflatten_ok(self): + import colander + fstruct = { + 'schema.seq.2.tupstring': 's', + 'schema.seq2.0.key2': 2, + 'schema.ob': colander.tests, + 'schema.seq2.1.key2': 4, + 'schema.seq.1.tupstring': 's', + 'schema.seq2.0.key': 1, + 'schema.seq.1.tupint': 2, + 'schema.seq.0.tupstring': 's', + 'schema.seq.3.tupstring': 's', + 'schema.seq.3.tupint': 4, + 'schema.seq2.1.key': 3, + 'schema.int': 10, + 'schema.seq.0.tupint': 1, + 'schema.tup.tupint': 1, + 'schema.tup.tupstring': 's', + 'schema.seq.2.tupint': 3, + } + schema = self._makeSchema() + result = schema.unflatten(fstruct) + + expected = { + 'int':10, + 'ob':colander.tests, + 'seq':[(1, 's'),(2, 's'), (3, 's'), (4, 's')], + 'seq2':[{'key':1, 'key2':2}, {'key':3, 'key2':4}], + 'tup':(1, 's'), + } + + for k, v in expected.items(): + self.assertEqual(result[k], v) + for k, v in result.items(): + self.assertEqual(expected[k], v) + + def test_flatten_unflatten_roundtrip(self): + import colander + appstruct = { + 'int':10, + 'ob':colander.tests, + 'seq':[(1, 's'),(2, 's'), (3, 's'), (4, 's')], + 'seq2':[{'key':1, 'key2':2}, {'key':3, 'key2':4}], + 'tup':(1, 's'), + } + schema = self._makeSchema(name='') + self.assertEqual( + schema.unflatten(schema.flatten(appstruct)), + appstruct) + def test_invalid_asdict(self): expected = { 'schema.int': '20 is greater than maximum value 10', @@ -2001,8 +2142,8 @@ class TestFunctional(object): self.assertEqual(errors, expected) class TestImperative(unittest.TestCase, TestFunctional): - - def _makeSchema(self): + + def _makeSchema(self, name='schema'): import colander integer = colander.SchemaNode( @@ -2059,14 +2200,13 @@ class TestImperative(unittest.TestCase, TestFunctional): tup, seq, seq2, - name='schema') + name=name) return schema - class TestDeclarative(unittest.TestCase, TestFunctional): - - def _makeSchema(self): + + def _makeSchema(self, name='schema'): import colander @@ -2092,7 +2232,7 @@ class TestDeclarative(unittest.TestCase, TestFunctional): tup = TupleSchema() seq2 = SequenceTwo() - schema = MainSchema(name='schema') + schema = MainSchema(name=name) return schema class Test_null(unittest.TestCase): @@ -2133,6 +2273,11 @@ class DummySchemaNode(object): raise Invalid(self, self.exc) return val + def __getitem__(self, name): + for child in self.children: + if child.name == name: + return child + class DummyValidator(object): def __init__(self, msg=None): self.msg = msg @@ -2147,7 +2292,7 @@ class Uncooperative(object): raise ValueError('I wont cooperate') __unicode__ = __str__ - + class DummyType(object): def serialize(self, node, value): return value @@ -2155,7 +2300,13 @@ class DummyType(object): def deserialize(self, node, value): return value - def flatten(self, node, appstruct, prefix=''): - key = prefix + 'appstruct' + def flatten(self, node, appstruct, prefix='', listitem=False): + if listitem: + key = prefix.rstrip('.') + else: + key = prefix + 'appstruct' return {key:appstruct} + def unflatten(self, node, paths, fstruct): + assert paths == [node.name] + return fstruct[node.name]