diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 00000000..cbdc2ffc --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +# +# tooz documentation build configuration file +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import datetime +import subprocess + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.todo', + 'sphinx.ext.graphviz', + 'sphinx.ext.extlinks', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'tooz' +copyright = u'%s, eNovance' % datetime.date.today().year + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = subprocess.Popen(['sh', '-c', 'cd ../..; python setup.py --version'], + stdout=subprocess.PIPE).stdout.read() +version = version.strip() +# The full version, including alpha/beta/rc tags. +release = version + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'toozdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'tooz.tex', u'tooz Documentation', + u'eNovance', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +# man_pages = [ +# ('index', 'tooz', u'tooz Documentation', +# [u'eNovance'], 1) +# ] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'tooz', u'tooz Documentation', + u'eNovance', 'tooz', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# extlinks = { +# } + +autodoc_default_flags = ['members', 'special-members', 'show-inheritance'] diff --git a/doc/source/drivers.rst b/doc/source/drivers.rst new file mode 100644 index 00000000..34971f7a --- /dev/null +++ b/doc/source/drivers.rst @@ -0,0 +1,23 @@ +======= +Drivers +======= + +Tooz is provided with several drivers implementing the provided coordination +API. While all drivers provides the same set of features with respect to the +API, some of them have different properties: + +* `zookeeper`_ is the reference implementation and provides the most solid + features as it's possible to build a cluster of ZooKeeper that is + resilient towards network partitions for example. + +* `memcached`_ is a basic implementation and provides less resiliency, though + it's much simpler to setup. A lot of the features provided in tooz are + based on timeout (heartbeats, locks, etc) so are less resilient than other + backends. + +* `zake`_ is a driver using a fake implementation of ZooKeeper and can be + used to use Tooz in your unit tests suite for example. + +.. _zookeeper: http://zookeeper.apache.org/ +.. _memcached: http://memcached.org/ +.. _zake: https://pypi.python.org/pypi/zake diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 00000000..da2ae549 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,22 @@ +============================================================= + tooz -- Distributed System Helper Library +============================================================= + +The Tooz project aims at centralizing the most common distributed primitives +like group membership protocol, lock service and leader election by providing +a coordination API helping developers to build distributed applications. +Contents: + +.. toctree:: + :glob: + + install + drivers + tutorial/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/source/install.rst b/doc/source/install.rst new file mode 100644 index 00000000..0e72b7b9 --- /dev/null +++ b/doc/source/install.rst @@ -0,0 +1,43 @@ +============ +Installation +============ + +Python Versions +=============== + +tooz is tested under Python 2.7 and 3.3. + +.. _install-basic: + +Basic Installation +================== + +tooz should be installed into the same site-packages area where +the application and extensions are installed (either a virtualenv or +the global site-packages). You may need administrative privileges to +do that. The easiest way to install it is using pip_:: + + $ pip install tooz + +or:: + + $ sudo pip install tooz + +.. _pip: http://pypi.python.org/pypi/pip + +Download +======== + +tooz releases are hosted on PyPI and can be downloaded from: +http://pypi.python.org/pypi/tooz + +Source Code +=========== + +The source is hosted on the OpenStack infrastructure: https://git.openstack.org/cgit/stackforge/tooz/ + +Reporting Bugs +============== + +Please report bugs through the launchpad project: +https://launchpad.net/python-tooz diff --git a/doc/source/tutorial/coordinator.rst b/doc/source/tutorial/coordinator.rst new file mode 100644 index 00000000..23d39cc6 --- /dev/null +++ b/doc/source/tutorial/coordinator.rst @@ -0,0 +1,42 @@ +===================== + Creating A Coordinator +===================== + +The principal object provided by tooz is the *coordinator*. It allows you to +use various features, such as group membership, leader election or +distributed locking. + +The features provided by tooz coordinator are implemented using different +drivers. When creating a coordinator, you need to specify which back-end +driver you want it to use. Different drivers may provide different set of +capabilities. + +If a driver does not support a feature, it will raise a +:class:`~NotImplementedError` exception. + +This example program loads a basic coordinataor using the ZooKeeper based +driver. + +.. literalinclude:: ../../../examples/coordinator.py + :language: python + +The second argument passed to the coordinator must be a unique identifier +identifying the running program. + +After the coordinator is created, it can be used to use the various features +provided. + +In order to keep the connection to the coordination server active, you must +call regularly the :meth:`~tooz.coordination.CoordinationDriver.heartbeat` +method. This will ensure that the coordinator is not considered dead by +other program participating in the coordination. + +.. literalinclude:: ../../../examples/coordinator_heartbeat.py + :language: python + +We use a pretty simple mechanism in this example to send a heartbeat every +once in a while, but depending on your application, you may want to send the +heartbeat at different moment or intervals. + +Note that certain drivers, such as `memcached` are heavily based on timeout, +so the interval used to run the heartbeat is important. diff --git a/doc/source/tutorial/group_membership.rst b/doc/source/tutorial/group_membership.rst new file mode 100644 index 00000000..d16da8c3 --- /dev/null +++ b/doc/source/tutorial/group_membership.rst @@ -0,0 +1,41 @@ +===================== + Group Membership +===================== + +Basic operations +=================== + +One of the feature provided by the coordinator is the ability to handle +group membership. Once a group is created, any coordinator can join the +group and become a member of it. Any coordinator can be notified when a +members joins or leaves the group. + +.. literalinclude:: ../../../examples/group_membership.py + :language: python + +Note that all the operation are asynchronous. That means you cannot be sure +that your group has been created or joined before you call the +:meth:`tooz.coordination.CoordAsyncResult.get` method. + +You can also leave a group using the +:meth:`tooz.coordination.CoordinationDriver.leave_group` method. The list of +all available groups is retrievable via the +:meth:`tooz.coordination.CoordinationDriver.get_groups` method. + +Watching Group Changes +====================== +It's possible to watch and get notified when the member list of a group +changes. That's useful to run callback functions whenever something happens +in that group. + + +.. literalinclude:: ../../../examples/group_membership_watch.py + :language: python + +Using :meth:`tooz.coordination.CoordinationDriver.watch_join_group` and +:meth:`tooz.coordination.CoordinationDriver.watch_leave_group` your +application can be notified each time a member join or leave a group. To +stop watching an event, the two methods +:meth:`tooz.coordination.CoordinationDriver.unwatch_join_group` and +:meth:`tooz.coordination.CoordinationDriver.unwatch_leave_group` allow to +unregister a particular callback. diff --git a/doc/source/tutorial/index.rst b/doc/source/tutorial/index.rst new file mode 100644 index 00000000..e83c0fc1 --- /dev/null +++ b/doc/source/tutorial/index.rst @@ -0,0 +1,13 @@ +===================================== + Using Tooz in Your Application +===================================== + +This tutorial is a step-by-step walk-through demonstrating how to +use tooz in your application. + +.. toctree:: + :maxdepth: 2 + + coordinator + group_membership + leader_election diff --git a/doc/source/tutorial/leader_election.rst b/doc/source/tutorial/leader_election.rst new file mode 100644 index 00000000..052e6c35 --- /dev/null +++ b/doc/source/tutorial/leader_election.rst @@ -0,0 +1,25 @@ +================= + Leader Election +================= + +Each group can elect its own leader. There can be only one leader at a time +in a group. Only members that are running for the election can be elected. +As soon as one of leader steps down or dies, a new member that was running +for the election will be elected. + +.. literalinclude:: ../../../examples/leader_election.py + :language: python + +The method +:meth:`tooz.coordination.CoordinationDriver.watch_elected_as_leader` allows +to register for a function to be called back when the member is elected as a +leader. Using this function indicates that the run is therefore running for +the election. The member can stop running by unregistering all its callbacks +with :meth:`tooz.coordination.CoordinationDriver.unwatch_elected_as_leader`. +It can also temporarily try to step down as a leader with +:meth:`tooz.coordination.CoordinationDriver.stand_down_group_leader`. If +another member is in the run for election, it may be elected instead. + +To retrieve the leader of a group, even when not being part of the group, +the method :meth:`tooz.coordination.CoordinationDriver.get_leader()` can be +used. diff --git a/examples/coordinator.py b/examples/coordinator.py new file mode 100644 index 00000000..c6d07925 --- /dev/null +++ b/examples/coordinator.py @@ -0,0 +1,5 @@ +from tooz import coordination + +coordinator = coordination.get_coordinator('zookeeper', b'host-1') +coordinator.start() +coordinator.stop() diff --git a/examples/coordinator_heartbeat.py b/examples/coordinator_heartbeat.py new file mode 100644 index 00000000..45539eb5 --- /dev/null +++ b/examples/coordinator_heartbeat.py @@ -0,0 +1,12 @@ +import time + +from tooz import coordination + +coordinator = coordination.get_coordinator('memcached', b'host-1') +coordinator.start() + +while True: + coordinator.heartbeat() + time.sleep(0.1) + +coordinator.stop() diff --git a/examples/group_membership.py b/examples/group_membership.py new file mode 100644 index 00000000..a94b5e57 --- /dev/null +++ b/examples/group_membership.py @@ -0,0 +1,14 @@ +from tooz import coordination + +coordinator = coordination.get_coordinator('zookeeper', b'host-1') +coordinator.start() + +# Create a group +request = coordinator.create_group(b"my group") +request.get() + +# Join a group +request = coordinator.join_group(b"my group") +request.get() + +coordinator.stop() diff --git a/examples/group_membership_watch.py b/examples/group_membership_watch.py new file mode 100644 index 00000000..4da1963c --- /dev/null +++ b/examples/group_membership_watch.py @@ -0,0 +1,18 @@ +from tooz import coordination + +coordinator = coordination.get_coordinator('zookeeper', b'host-1') +coordinator.start() + +# Create a group +request = coordinator.create_group(b"my group") +request.get() + + +def group_joined(event): + # Event is an instance of tooz.coordination.MemberJoinedGroup + print(event.group_id, event.member_id) + + +coordinator.watch_join_group(b"my group", group_joined) + +coordinator.stop() diff --git a/examples/leader_election.py b/examples/leader_election.py new file mode 100644 index 00000000..2daea4aa --- /dev/null +++ b/examples/leader_election.py @@ -0,0 +1,28 @@ +from tooz import coordination + +coordinator = coordination.get_coordinator('zookeeper', b'host-1') +coordinator.start() + +# Create a group +request = coordinator.create_group(b"my group") +request.get() + +# Join a group +request = coordinator.join_group(b"my group") +request.get() + + +def when_i_am_elected_leader(event): + # event is a LeaderElected event + print(event.group_id, event.member_id) + + +# Propose to be a leader for the group +coordinator.watch_elected_as_leader(b"my_group", + when_i_am_elected_leader) + +while True: + coordinator.heartbeat() + coordinator.run_watchers() + +coordinator.stop() diff --git a/setup.cfg b/setup.cfg index 126fb392..787ac2c7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,3 +27,8 @@ tooz.backends = kazoo = tooz.drivers.zookeeper:KazooDriver zake = tooz.drivers.zookeeper:ZakeDriver memcached = tooz.drivers.memcached:MemcachedDriver + +[build_sphinx] +all_files = 1 +build-dir = doc/build +source-dir = doc/source diff --git a/test-requirements.txt b/test-requirements.txt index 18a1e649..ef7251eb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,6 +2,7 @@ pep8>=1.4.5 pyflakes>=0.7.2,<0.7.4 flake8>=2.1.0 discover +sphinx>=1.1.2,<1.2 python-subunit testrepository>=0.0.17 testtools>=0.9.32 diff --git a/tox.ini b/tox.ini index 7baecacd..bf760f4b 100644 --- a/tox.ini +++ b/tox.ini @@ -24,5 +24,5 @@ commands = [flake8] ignore = H405 -exclude=.venv,.git,.tox,dist,*egg,*.egg-info,build +exclude=.venv,.git,.tox,dist,*egg,*.egg-info,build,examples,doc show-source = True