Add partial work towards schema / validation / serialization / deserialization thingy.

This commit is contained in:
Chris McDonough
2010-03-11 08:55:25 +00:00
commit 35fb6f5b28
14 changed files with 870 additions and 0 deletions

7
CHANGES.txt Normal file
View File

@@ -0,0 +1,7 @@
Changes
=======
0.0 (unreleased)
----------------
- Initial release.

3
COPYRIGHT.txt Normal file
View File

@@ -0,0 +1,3 @@
Copyright (c) 2010 Agendaless Consulting and Contributors.
(http://www.agendaless.com), All Rights Reserved

41
LICENSE.txt Normal file
View File

@@ -0,0 +1,41 @@
License
A copyright notice accompanies this license document that identifies
the copyright holders.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions in source code must retain the accompanying
copyright notice, this list of conditions, and the following
disclaimer.
2. Redistributions in binary form must reproduce the accompanying
copyright notice, this list of conditions, and the following
disclaimer in the documentation and/or other materials provided
with the distribution.
3. Names of the copyright holders must not be used to endorse or
promote products derived from this software without prior
written permission from the copyright holders.
4. If any files are modified, you must cause the modified files to
carry prominent notices stating that you changed the files and
the date of any change.
Disclaimer
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND
ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH DAMAGE.

12
README.txt Normal file
View File

@@ -0,0 +1,12 @@
cereal
======
An extensible package which can be used to:
- deserialize and validate a data structure composed of strings,
mappings, and lists.
- serialize an arbitrary data structure to a data structure composed
of strings, mappings, and lists.
Please see docs/index.rst for further documentation.

371
cereal/__init__.py Normal file
View File

@@ -0,0 +1,371 @@
import pkg_resources
def resolve_dotted(dottedname, package=None):
if dottedname.startswith('.') or dottedname.startswith(':'):
if not package:
raise ImportError('name "%s" is irresolveable (no package)' %
dottedname)
if dottedname in ['.', ':']:
dottedname = package.__name__
else:
dottedname = package.__name__ + dottedname
return pkg_resources.EntryPoint.parse(
'x=%s' % dottedname).load(False)
class Invalid(Exception):
pos = None
parent = None
def __init__(self, struct, msg=None):
Exception.__init__(self, struct, msg)
self.struct = struct
self.msg = msg
self.subexceptions = []
def add(self, error):
if self.msg is not None:
raise ValueError(
'Exceptions with a message cannot have subexceptions')
error.parent = self
self.subexceptions.append(error)
def expand(self):
L = []
L.append(self.msg)
for exc in self.subexceptions:
L.append((exc.pos, self.struct, exc.expand()))
return L
def pprint(self, indent=0):
istring = ' ' * indent
for exc in self.subexceptions:
if exc.msg:
print '%s%s (%s): %s' % (
istring, exc.struct.name, exc.pos, exc.msg)
else:
print '%s%s (%s):' % (istring, exc.struct.name, exc.pos)
exc.pprint(indent+2)
def paths(self):
# thanks chris rossi ;-)
def traverse(node, stack):
stack.append(node)
if not node.subexceptions:
yield tuple(stack)
for child in node.subexceptions:
for path in traverse(child, stack):
yield path
stack.pop()
return traverse(self, [])
def asdict(self):
paths = list(self.paths())
D = {}
for path in paths:
L = []
msg = None
for exc in path:
msg = exc.msg
if exc.parent is not None:
if isinstance(exc.parent.struct.typ, Positional):
L.append(str(exc.pos))
else:
L.append(exc.struct.name)
D['.'.join(L)] = msg
return D
class All(object):
def __init__(self, *validators):
self.validators = validators
def __call__(self, struct, value):
msgs = []
for validator in self.validators:
try:
validator(struct, value)
except Invalid, e:
msgs.append(e.msg)
if msgs:
raise Invalid(struct, msgs)
class Range(object):
def __init__(self, min=None, max=None):
self.min = min
self.max = max
def __call__(self, struct, value):
if self.min is not None:
if value < self.min:
raise Invalid(
struct,
'%r is less than minimum value %r' % (value, self.min))
if self.max is not None:
if value > self.max:
raise Invalid(
struct,
'%r is greater than maximum value %r' % (value, self.max))
class Mapping(object):
def _validate(self, struct, value):
if not isinstance(value, dict):
raise Invalid(struct, '%r is not a mapping type' % value)
return value
def serialize(self, struct, value):
value = self._validate(struct, value)
result = {}
error = None
for num, substruct in enumerate(struct.structs):
name = substruct.name
subval = value.get(name)
try:
if subval is None:
if substruct.required and substruct.default is None:
raise Invalid(
substruct,
'%r is required but empty' % substruct.name)
result[name] = substruct.serialize(struct.default)
else:
result[name] = substruct.serialize(subval)
except Invalid, e:
if error is None:
error = Invalid(substruct)
e.pos = num
error.add(e)
if error is not None:
raise error
return result
def deserialize(self, struct, value):
value = self._validate(struct, value)
error = None
result = {}
for num, substruct in enumerate(struct.structs):
name = substruct.name
subval = value.get(name)
try:
if subval is None:
if substruct.required and substruct.default is None:
raise Invalid(
substruct,
'%r is required but empty' % substruct.name)
result[name] = substruct.default
else:
result[name] = substruct.deserialize(subval)
except Invalid, e:
if error is None:
error = Invalid(struct)
e.pos = num
error.add(e)
if error is not None:
raise error
return result
class Positional(object):
"""
Marker abstract base class meaning 'this type has children which
should be addressed by position instead of name' (e.g. via seq[0],
but never seq['name']). This is consulted by Invalid.asdict when
creating a dictionary representation of an error structure.
"""
class Tuple(Positional):
""" A type which represents a fixed-length sequence of data
structures, each one of which may be different as denoted by the
types of the associated structure's children."""
def _validate(self, struct, value):
if not hasattr(value, '__iter__'):
raise Invalid(struct, '%r is not an iterable value' % value)
return list(value)
def serialize(self, struct, value):
value = self._validate(struct, value)
error = None
result = []
for num, substruct in enumerate(struct.structs):
try:
subval = value[num]
except IndexError:
raise Invalid(struct, 'Wrong number of elements in %r' % value)
try:
result.append(substruct.serialize(subval))
except Invalid, e:
if error is None:
error = Invalid(struct)
e.pos = num
e.sequence_child = True
error.add(e)
if error:
raise error
return tuple(result)
def deserialize(self, struct, value):
value = self._validate(struct, value)
error = None
result = []
for num, substruct in enumerate(struct.structs):
try:
subval = value[num]
except IndexError:
raise Invalid(struct, 'Wrong number of elements in %r' % value)
try:
result.append(substruct.deserialize(subval))
except Invalid, e:
if error is None:
error = Invalid(struct)
e.pos = num
e.sequence_child = True
error.add(e)
if error:
raise error
return tuple(result)
class Sequence(Positional):
""" A type which represents a variable-length sequence of values,
all of which must be of the same type as denoted by the type of
``substruct``"""
def __init__(self, substruct):
self.substruct = substruct
def _validate(self, struct, value):
if not hasattr(value, '__iter__'):
raise Invalid(struct, '%r is not an iterable value' % value)
return list(value)
def serialize(self, struct, value):
value = self._validate(struct, value)
error = None
result = []
for num, subval in enumerate(value):
try:
result.append(self.substruct.serialize(subval))
except Invalid, e:
if error is None:
error = Invalid(struct)
e.pos = num
error.add(e)
if error:
raise error
return result
def deserialize(self, struct, value):
value = self._validate(struct, value)
error = None
result = []
for num, sub in enumerate(value):
try:
result.append(self.substruct.deserialize(sub))
except Invalid, e:
if error is None:
error = Invalid(struct)
e.pos = num
error.add(e)
if error:
raise error
return result
class String(object):
""" A type representing a Unicode string """
def __init__(self, encoding='utf-8'):
self.encoding = encoding
def _validate(self, struct, value):
try:
if isinstance(value, unicode):
return value
return unicode(value, self.encoding)
except:
raise Invalid(struct, '%r is not a string' % value)
def serialize(self, struct, value):
decoded = self._validate(struct, value)
return decoded.encode(struct.encoding)
def deserialize(self, struct, value):
return self._validate(struct, value)
class Integer(object):
""" A type representing an integer """
def _validate(self, struct, value):
try:
return int(value)
except:
raise Invalid(struct, '%r is not a number' % value)
def serialize(self, struct, value):
return str(self._validate(struct, value))
def deserialize(self, struct, value):
return self._validate(struct, value)
class GlobalObject(object):
""" A type representing an importable Python object """
def __init__(self, package):
self.package = package
def serialize(self, struct, value):
try:
return value.__name__
except AttributeError:
raise Invalid(struct, '%r has no __name__' % value)
def deserialize(self, struct, value):
if not isinstance(value, basestring):
raise Invalid(struct, '%r is not a global object specification')
try:
return resolve_dotted(value, package=self.package)
except ImportError:
raise Invalid(struct,
'The dotted name %r cannot be imported' % value)
class Structure(object):
def __init__(self, name, typ, validator=None, default=None, required=True):
self.typ = typ
self.name = name
self.validator = validator
self.default = default
self.required = required
self.structs = []
def serialize(self, value):
return self.typ.serialize(self, value)
def deserialize(self, value):
value = self.typ.deserialize(self, value)
if self.validator is not None:
self.validator(self, value)
return value
def add(self, struct):
self.structs.append(struct)

76
cereal/tests.py Normal file
View File

@@ -0,0 +1,76 @@
import unittest
class TestFunctional(unittest.TestCase):
def _makeSchema(self):
import cereal
integer = cereal.Structure(
'int', cereal.Integer(), validator=cereal.Range(0, 10))
ob = cereal.Structure('ob', cereal.GlobalObject(package=cereal))
tup = cereal.Structure('tup', cereal.Tuple())
tup.add(cereal.Structure('tupint', cereal.Integer()))
tup.add(cereal.Structure('tupstring', cereal.String()))
seq = cereal.Structure('seq', cereal.Sequence(tup))
mapping = cereal.Structure('mapping', cereal.Mapping())
seq2 = cereal.Structure('seq2', cereal.Sequence(mapping))
mapping.add(cereal.Structure('key', cereal.Integer()))
mapping.add(cereal.Structure('key2', cereal.Integer()))
schema = cereal.Structure('', cereal.Mapping())
schema.add(integer)
schema.add(ob)
schema.add(tup)
schema.add(seq)
schema.add(seq2)
return schema
def test_deserialize_ok(self):
import cereal.tests
data = {
'int':'10',
'ob':'cereal.tests',
'tup':('1', 's'),
'seq':[('1', 's'),('2', 's'), ('3', 's'), ('4', 's')],
'seq2':[{'key':'1', 'key2':'2'}, {'key':'3', 'key2':'4'}],
}
schema = self._makeSchema()
result = schema.deserialize(data)
self.assertEqual(result['int'], 10)
self.assertEqual(result['ob'], cereal.tests)
self.assertEqual(result['tup'], (1, 's'))
self.assertEqual(result['seq'],
[(1, 's'), (2, 's'), (3, 's'), (4, 's')])
self.assertEqual(result['seq2'],
[{'key':1, 'key2':2}, {'key':3, 'key2':4}])
def test_invalid_asdict(self):
expected = {
'int': '20 is greater than maximum value 10',
'ob': "The dotted name 'no.way.this.exists' cannot be imported",
'seq.0.0': "'q' is not a number",
'seq.1.0': "'w' is not a number",
'seq.2.0': "'e' is not a number",
'seq.3.0': "'r' is not a number",
'seq2.0.key': "'t' is not a number",
'seq2.0.key2': "'y' is not a number",
'seq2.1.key': "'u' is not a number",
'seq2.1.key2': "'i' is not a number",
'tup.0': "'s' is not a number"}
import cereal
data = {
'int':'20',
'ob':'no.way.this.exists',
'tup':('s', 's'),
'seq':[('q', 's'),('w', 's'), ('e', 's'), ('r', 's')],
'seq2':[{'key':'t', 'key2':'y'}, {'key':'u', 'key2':'i'}],
}
schema = self._makeSchema()
try:
schema.deserialize(data)
except cereal.Invalid, e:
errors = e.asdict()
self.assertEqual(errors, expected)

BIN
docs/.static/logo_hi.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.8 KiB

22
docs/.static/repoze.css Normal file
View File

@@ -0,0 +1,22 @@
@import url('default.css');
body {
background-color: #006339;
}
div.document {
background-color: #dad3bd;
}
div.sphinxsidebar h3, h4, h5, a {
color: #127c56 !important;
}
div.related {
color: #dad3bd !important;
background-color: #00744a;
}
div.related a {
color: #dad3bd !important;
}

70
docs/Makefile Normal file
View File

@@ -0,0 +1,70 @@
# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d .build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help clean html web pickle htmlhelp latex changes linkcheck
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " pickle to make pickle files (usable by e.g. sphinx-web)"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " changes to make an overview over all changed/added/deprecated items"
@echo " linkcheck to check all external links for integrity"
clean:
-rm -rf .build/*
html:
mkdir -p .build/html .build/doctrees
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) .build/html
@echo
@echo "Build finished. The HTML pages are in .build/html."
pickle:
mkdir -p .build/pickle .build/doctrees
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) .build/pickle
@echo
@echo "Build finished; now you can process the pickle files or run"
@echo " sphinx-web .build/pickle"
@echo "to start the sphinx-web server."
web: pickle
htmlhelp:
mkdir -p .build/htmlhelp .build/doctrees
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) .build/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in .build/htmlhelp."
latex:
mkdir -p .build/latex .build/doctrees
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) .build/latex
@echo
@echo "Build finished; the LaTeX files are in .build/latex."
@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
"run these through (pdf)latex."
changes:
mkdir -p .build/changes .build/doctrees
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) .build/changes
@echo
@echo "The overview file is in .build/changes."
linkcheck:
mkdir -p .build/linkcheck .build/doctrees
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) .build/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in .build/linkcheck/output.txt."

4
docs/api.rst Normal file
View File

@@ -0,0 +1,4 @@
API Documentation
=================
XXX

185
docs/conf.py Normal file
View File

@@ -0,0 +1,185 @@
# -*- coding: utf-8 -*-
#
# cereal documentation build configuration file
#
# This file is execfile()d with the current directory set to its containing
# dir.
#
# The contents of this file are pickled, so don't put values in the
# namespace that aren't pickleable (module imports are okay, they're
# removed automatically).
#
# All configuration values have a default value; values that are commented
# out serve to show the default value.
import sys, os
# If your extensions are in another directory, add it here. If the
# directory is relative to the documentation root, use os.path.abspath to
# make it absolute, like shown here.
#sys.path.append(os.path.abspath('some/directory'))
# General configuration
# ---------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['.templates']
# The suffix of source filenames.
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# General substitutions.
project = 'cereal'
copyright = '2010, Repoze Developers <repoze-dev@lists.repoze.org>'
# The default replacements for |version| and |release|, also used in various
# other places throughout the built documents.
#
# The short X.Y version.
version = '0.0'
# The full version, including alpha/beta/rc tags.
release = '0.0'
# There are two options for replacing |today|: either, you set today to
# some non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
today_fmt = '%B %d, %Y'
# List of documents that shouldn't be included in the build.
#unused_docs = []
# List of directories, relative to source directories, that shouldn't be
# searched for source files.
#exclude_dirs = []
# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# Options for HTML output
# -----------------------
# The style sheet to use for HTML and HTML Help pages. A file of that name
# must exist either in Sphinx' static/ path, or in one of the custom paths
# given in html_static_path.
html_style = 'repoze.css'
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# A shorter title for the navigation bar. Default is the same as
# html_title.
#html_short_title = None
# The name of an image file (within the static path) to place at the top of
# the sidebar.
html_logo = '.static/logo_hi.gif'
# The name of an image file (within the static path) to use as favicon of
# the docs. This file should be a Windows icon file (.ico) being 16x16 or
# 32x32 pixels large.
#html_favicon = None
# Add any paths that contain custom static files (such as style sheets)
# here, relative to this directory. They are copied after the builtin
# static files, so a file named "default.css" will overwrite the builtin
# "default.css".
html_static_path = ['.static']
# If not '', a 'Last updated on:' timestamp is inserted at every page
# bottom, using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# If false, no module index is generated.
#html_use_modindex = True
# If false, no index is generated.
#html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# If true, the reST sources are included in the HTML build as
# _sources/<name>.
#html_copy_source = True
# If true, an OpenSearch description file will be output, and all pages
# will contain a <link> tag referring to it. The value of this option must
# be the base URL from which the finished HTML is served.
#html_use_opensearch = ''
# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''
# Output file base name for HTML help builder.
htmlhelp_basename = 'atemplatedoc'
# Options for LaTeX output
# ------------------------
# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'
# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, document class [howto/manual]).
latex_documents = [
('index', 'atemplate.tex', 'cereal Documentation',
'Repoze Developers', 'manual'),
]
# The name of an image file (relative to this directory) to place at the
# top of the title page.
latex_logo = '.static/logo_hi.gif'
# For "manual" documents, if this is true, then toplevel headings are
# parts, not chapters.
#latex_use_parts = False
# Additional stuff for the LaTeX preamble.
#latex_preamble = ''
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# If false, no module index is generated.
#latex_use_modindex = True

23
docs/index.rst Normal file
View File

@@ -0,0 +1,23 @@
cereal
======
Cereal is an extensible package which can be used to:
- deserialize and validate a data structure composed of strings,
mappings, and lists.
- serialize an arbitrary data structure to a data structure composed
of strings, mappings, and lists.
.. toctree::
:maxdepth: 2
api.rst
Indices and tables
------------------
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

10
setup.cfg Normal file
View File

@@ -0,0 +1,10 @@
[easy_install]
zip_ok = false
[nosetests]
match=^test
where=cereal
nocapture=1
cover-package=cereal
cover-erase=1

46
setup.py Normal file
View File

@@ -0,0 +1,46 @@
##############################################################################
#
# Copyright (c) 2010 Agendaless Consulting and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the BSD-like license at
# http://www.repoze.org/LICENSE.txt. A copy of the license should accompany
# this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL
# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND
# FITNESS FOR A PARTICULAR PURPOSE
#
##############################################################################
import os
from setuptools import setup
from setuptools import find_packages
here = os.path.abspath(os.path.dirname(__file__))
README = open(os.path.join(here, 'README.txt')).read()
CHANGES = open(os.path.join(here, 'CHANGES.txt')).read()
requires = []
setup(name='cereal',
version='0.0',
description='A schema-based serialization and deserialization library',
long_description=README + '\n\n' + CHANGES,
classifiers=[
"Intended Audience :: Developers",
"Programming Language :: Python",
],
keywords='serialize deserialize validate schema',
author="Agendaless Consulting",
author_email="repoze-dev@lists.repoze.org",
url="http://www.repoze.org",
license="BSD-derived (http://www.repoze.org/LICENSE.txt)",
packages=find_packages(),
include_package_data=True,
zip_safe=False,
tests_require = requires,
install_requires = requires,
test_suite="cereal",
)