Merge branch 'add-tutorial'

.gitignore

@@ -25,3 +25,6 @@ pip-log.txt
 
 #Mr Developer
 .mr.developer.cfg
+
+# Editors
+tags

docs/source/history.rst

@@ -15,6 +15,7 @@ dev
 - Add ``__getitem__`` to
   :class:`~stevedore.extension.ExtensionManager` for looking up
   individual plugins by name (:issue:`15`).
+- Start working on the tutorial, :doc:`tutorial/index`.
 
 0.8
 

docs/source/index.rst

@@ -19,6 +19,7 @@ Contents:
 
    patterns_loading
    patterns_enabling
+   tutorial/index
    managers
    install
    essays/*

docs/source/tutorial/creating_plugins.rst

@@ -0,0 +1,125 @@
+==================
+ Creating Plugins
+==================
+
+After a lot of trial and error, the easiest way I have found to define
+an API is to follow these steps:
+
+#. Use the `abc module`_ to create a base abstract class to define the
+   behaviors required of plugins of the API.  Developers don't have to
+   subclass from the base class, but it provides a convenient way to
+   document the API, and using an abstract base class keeps you
+   honest.
+#. Create plugins by subclassing the base class and implementing the
+   required methods.
+#. Define a unique namespace for each API by combining the name of the
+   application (or library) and a name of the API. Keep it
+   shallow. For example, "cliff.formatters" or
+   "ceilometer.pollsters.compute".
+
+Example Plugin Set
+==================
+
+The example program in this tutorial will create a plugin set with
+several data formatters, like what might be used by a command line
+program to prepare data to be printed to the console.  Each formatter
+will take as input a dictionary with string keys and built-in data
+types as values. It will return as output an iterator that produces
+the string with the data structure formatted based on the rules of the
+specific formatter being used. The formatter's constructor lets the
+caller specify the maximum width the output should have.
+
+A Plugin Base Class
+===================
+
+Step 1 above is to define an abstract base class for the API that
+needs to be implemented by each plugin. 
+
+.. literalinclude:: ../../../stevedore/example/base.py
+   :language: python
+   :prepend: # stevedore/example/base.py
+
+The constructor is a concrete method because subclasses do not need to
+override it, but the :func:`format` method does not do anything useful
+because there is no "default" implementation available.
+
+Concrete Plugins
+================
+
+The next step is to create a couple of plugin classes with concrete
+implementations of :func:`format`. A simple example formatter produces
+output with each variable name and value on a single line.
+
+.. literalinclude:: ../../../stevedore/example/simple.py
+   :language: python
+   :prepend: # stevedore/example/simple.py
+
+An alternate implementation produces a reStructuredText `field list`_.
+
+.. literalinclude:: ../../../stevedore/example/fields.py
+   :language: python
+   :prepend: # stevedore/example/fields.py
+
+There are plenty of other formatting options, but these two examples
+will give us enough to work with to demonstrate registering and using
+pluins.
+
+Registering the Plugins
+=======================
+
+To use setuptools entry points, you must package your application or
+library using setuptools. The build and packaging process generates
+metadata which is available after installation to find the plugins
+provided by each python distribution. 
+
+The entry points must be declared as belonging to a specific
+namespace, so we need to pick one before going any further. These
+plugins are formatters from the stevedore examples, so I will use the
+namespace "stevedore.example.formatter". Now it is possible to provide
+all of the necessary information in the packaging instructions:
+
+.. literalinclude:: ../../../stevedore/example/setup.py
+   :language: python
+   :prepend: # stevedore/example/setup.py
+
+The important lines are near the bottom where the ``entry_points``
+argument to :func:`setup` is set. The value is a dictionary mapping
+the namespace for the plugins to a list of their definitions. Each
+item in the list should be a string with ``name = module:importable``
+where *name* is the user-visible name for the plugin, *module* is the
+Python import reference for the module, and *importable* is the name
+of something that can be imported from inside the module.
+
+.. literalinclude:: ../../../stevedore/example/setup.py
+   :language: python
+   :lines: 37-43
+
+In this case, there are three plugins registered. The "simple" and
+"field" plugins defined above, and a "plain" plugin, which is just an
+alias for the simple plugin.
+
+setuptools Metadata
+===================
+
+During the build, setuptools copies entry point definitions to a file
+in the ".egg-info" directory for the package. For example, the file
+for stevedore is located in ``stevedore.egg-info/entry_points.txt``:
+
+::
+
+    [stevedore.example.formatter]
+    simple = stevedore.example.simple:Simple
+    field = stevedore.example.fields:FieldList
+    plain = stevedore.example.simple:Simple
+    
+    [stevedore.test.extension]
+    t2 = stevedore.tests.test_extension:FauxExtension
+    t1 = stevedore.tests.test_extension:FauxExtension
+
+:mod:`pkg_resources` uses the ``entry_points.txt`` file from all of
+the installed packages on the import path to find plugins. You should
+not modify these files, except by changing the list of entry points in
+``setup.py``.
+
+.. _abc module: http://docs.python.org/2/library/abc.html
+.. _field list: http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#field-lists

docs/source/tutorial/driver_output.txt

@@ -0,0 +1,36 @@
+$ python -m stevedore.example.load_as_driver a = A
+b = B
+long = word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word
+
+$ python -m stevedore.example.load_as_driver field
+: a : A
+: b : B
+: long : word word word word word word word word word word
+    word word word word word word word word word word word
+    word word word word word word word word word word word
+    word word word word word word word word word word word
+    word word word word word word word word word word word
+    word word word word word word word word word word word
+    word word word word word word word word word word word
+    word word word word
+
+$ python -m stevedore.example.load_as_driver field --width 30
+: a : A
+: b : B
+: long : word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word

docs/source/tutorial/extension_output.txt

@@ -0,0 +1,31 @@
+$ python -m stevedore.example.load_as_extension --width 30
+Formatter: simple
+a = A
+b = B
+long = word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word
+
+Formatter: field
+: a : A
+: b : B
+: long : word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word word word word word
+    word
+
+Formatter: plain
+a = A
+b = B
+long = word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word

docs/source/tutorial/index.rst

@@ -0,0 +1,26 @@
+=====================================
+ Using Stevedore in Your Application
+=====================================
+
+This tutorial is a step-by-step walk-through demonstrating how to
+define plugins and then use stevedore to load and use them in your
+application.
+
+.. toctree::
+   :maxdepth: 2
+
+   naming
+   creating_plugins
+   loading
+   testing
+
+.. seealso::
+
+   * :doc:`/essays/pycon2013`
+   * `Using setuptools entry points`_
+   * `Package Discovery and Resource Access using pkg_resources`_
+   * `Using Entry Points to Write Plugins | Pylons`_
+
+.. _Using setuptools entry points: http://reinout.vanrees.org/weblog/2010/01/06/zest-releaser-entry-points.html
+.. _Package Discovery and Resource Access using pkg_resources: http://pythonhosted.org/distribute/pkg_resources.html
+.. _Using Entry Points to Write Plugins | Pylons: http://docs.pylonsproject.org/projects/pylons-webframework/en/latest/advanced_pylons/entry_points_and_plugins.html

docs/source/tutorial/loading.rst

@@ -0,0 +1,126 @@
+=====================
+ Loading the Plugins
+=====================
+
+There are several different enabling and invocation patterns for
+consumers of plugins, depending on your needs.
+
+Loading Drivers
+===============
+
+The most common way plugins are used is as individual drivers. In this
+case, there may be many plugin options to choose from, but only one
+needs to be loaded and called. The
+:class:`~stevedore.driver.DriverManager` class supports this pattern.
+
+This example program uses a :class:`DriverManager` to load a formatter
+defined in the examples for stevedore. It then uses the formatter to
+convert a data structure to a text format, which it can print.
+
+.. literalinclude:: ../../../stevedore/example/load_as_driver.py
+   :language: python
+   :prepend: # stevedore/example/load_as_driver.py
+
+The manager takes the plugin namespace and name as arguments, and uses
+them to find the plugin. Then, because ``invoke_on_load`` is true, it
+calls the object loaded. In this case that object is the plugin class
+registered as a formatter. The ``invoke_args`` are positional
+arguments passed to the class constructor, and are used to set the
+maximum width parameter.
+
+.. literalinclude:: ../../../stevedore/example/load_as_driver.py
+   :language: python
+   :lines: 30-35
+
+After the manager is created, it holds a reference to a single object
+returned by calling the code registered for the plugin. That object is
+the actual driver, in this case an instance of the formatter class
+from the plugin. The single driver can be accessed via the
+:attr:`driver` property of the manager, and then its methods can be
+called directly.
+
+.. literalinclude:: ../../../stevedore/example/load_as_driver.py
+   :language: python
+   :lines: 36-37
+
+Running the example program produces this output:
+
+.. literalinclude:: driver_output.txt
+
+Loading Extensions
+==================
+
+Another common use case is to load several extensions at one time, and
+do something with all of them. Several of the other manager classes
+support this invocation pattern, including
+:class:`~stevedore.extension.ExtensionManager`,
+:class:`~stevedore.named.NamedExtensionManager`, and
+:class:`~stevedore.enabled.EnabledExtensionManager`.
+
+.. literalinclude:: ../../../stevedore/example/load_as_extension.py
+   :language: python
+   :prepend: # stevedore/example/load_as_extension.py
+
+The :class:`ExtensionManager` is created slightly differently from the
+:class:`DriverManager` because it does not need to know in advance
+which plugin to load. It loads all of the plugins it finds.
+
+.. literalinclude:: ../../../stevedore/example/load_as_extension.py
+   :language: python
+   :lines: 24-28
+
+To call the plugins, use the :meth:`map` method, passing a callable to
+be invoked for each extension. The :func:`format_data` function used
+with :meth:`map` in this example takes two arguments, the
+:class:`~stevedore.extension.Extension` and the data argument given to
+:meth:`map`.
+
+.. literalinclude:: ../../../stevedore/example/load_as_extension.py
+   :language: python
+   :lines: 30-33
+
+The :class:`Extension` passed :func:`format_data` is a class defined
+by stevedore that wraps the plugin. It includes the name of the
+plugin, the :class:`EntryPoint` returned by :mod:`pkg_resources`, and
+the plugin itself (the named object referenced by the plugin
+definition). When ``invoke_on_load`` is true, the :class:`Extension`
+will also have an :attr:`obj` attribute containing the value returned
+when the plugin was invoked.
+
+:meth:`map` returns a sequence of the values returned by the callback
+function. In this case, :func:`format_data` returns a tuple containing
+the extension name and the iterable that produces the text to
+print. As the results are processed, the name of each plugin is
+printed and then the formatted data.
+
+.. literalinclude:: ../../../stevedore/example/load_as_extension.py
+   :language: python
+   :lines: 35-39
+
+The order the plugins are loaded is undefined, and depends on the
+order packages are found on the import path as well as the way the
+metadata files are read. If the order extensions are used matters, try
+the :class:`~stevedore.named.NamedExtensionManager`.
+
+.. literalinclude:: extension_output.txt
+
+Why Not Call Plugins Directly?
+==============================
+
+Using a separate callable argument to :meth:`map`, rather than just
+invoking the plugin directly introduces a separation between your
+application code and the plugins. The benefits of this separation
+manifest in the application code design and in the plugin API design.
+
+If :meth:`map` called the plugin directly, each plugin would have to
+be a callable. That would mean a separate namespace for what is really
+just a method of the plugin. By using a separate callable argument,
+the plugin API does not need to match exactly any particular use case
+in the application. This frees you to create a finer-grained API, with
+more individual methods that can be called in different ways to
+achieve different goals.
+
+.. seealso::
+
+   * :doc:`/patterns_loading`
+   * :doc:`/patterns_enabling`

docs/source/tutorial/naming.rst

@@ -0,0 +1,38 @@
+===============================
+ Guidelines for Naming Plugins
+===============================
+
+Stevedore uses setuptools entry points to define and load plugins. An
+entry point is standard way to refer to a named object defined inside
+a Python module or package. The name can be a reference to any class,
+function, or instance, as long as it is created when the containing
+module is imported (i.e., it needs to be a module-level global).
+
+Names and Namespaces
+====================
+
+Entry points are registered using a *name* in a *namespace*. 
+
+Entry point names are usually considered user-visible. For example,
+they frequently appear in configuration files where a driver is being
+enabled.  Because they are public, names are typically as short as
+possible while remaining descriptive. For example, database driver
+plugin names might be "mysql", "postgresql", "sqlite", etc.
+
+Namespaces, on the other hand, are an implementation detail, and while
+they are known to developers they are not usually exposed to users.
+The namespace naming syntax looks a lot like Python's package syntax
+(``a.b.c``) but *namespaces do not correspond to Python
+packages*. Using a Python package name for an entry point namespace is
+an easy way to ensure a unique name, but it's not required at all.
+The main feature of entry points is that they can be discovered
+*across* packages. That means that a plugin can be developed and
+installed completely separately from the application that uses it, as
+long as they agree on the namespace and API.
+
+Each namespace is owned by the code that consumes the plugins and is
+used to search for entry points. The entry point names are typically
+owned by the plugin, but they can also be defined by the consuming
+code for named hooks (see :class:`~stevedore.hook.HookManager`).  The
+names of entry points must be unique within a given distribution, but
+are not necessarily unique in a namespace.

docs/source/tutorial/testing.rst

@@ -0,0 +1,9 @@
+=========
+ Testing
+=========
+
+.. describe using the TestManager for setting up application tests
+
+.. seealso::
+
+   * :class:`~stevedore.tests.manager.TestExtensionManager`

run_sphinx

@@ -4,7 +4,7 @@
 set -x
 
 watchmedo shell-command \
-    --patterns='*.rst;*.py' \
+    --patterns='*.rst;*.py;*.txt' \
     --ignore-pattern='docs/build/*;*flymake*' \
     --recursive \
     --command='python setup.py build_sphinx'

setup.py

@@ -52,6 +52,11 @@ setup(
     include_package_data=True,
 
     entry_points={
+        'stevedore.example.formatter': [
+            'simple = stevedore.example.simple:Simple',
+            'field = stevedore.example.fields:FieldList',
+            'plain = stevedore.example.simple:Simple',
+        ],
         'stevedore.test.extension': [
             't1 = stevedore.tests.test_extension:FauxExtension',
             't2 = stevedore.tests.test_extension:FauxExtension',

stevedore/example/base.py

@@ -0,0 +1,21 @@
+import abc
+
+
+class FormatterBase(object):
+    """Base class for example plugin used in the tutoral.
+    """
+
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, max_width=60):
+        self.max_width = max_width
+
+    @abc.abstractmethod
+    def format(self, data):
+        """Format the data and return unicode text.
+
+        :param data: A dictionary with string keys and simple types as
+                     values.
+        :type data: dict(str:?)
+        :returns: Iterable producing the formatted text.
+        """

stevedore/example/fields.py

@@ -0,0 +1,36 @@
+import textwrap
+
+from stevedore.example import base
+
+
+class FieldList(base.FormatterBase):
+    """Format values as a reStructuredText field list.
+
+    For example::
+
+      : name1 : value
+      : name2 : value
+      : name3 : a long value
+          will be wrapped with
+          a hanging indent
+    """
+
+    def format(self, data):
+        """Format the data and return unicode text.
+
+        :param data: A dictionary with string keys and simple types as
+                     values.
+        :type data: dict(str:?)
+        """
+        for name, value in sorted(data.items()):
+            full_text = ': {name} : {value}'.format(
+                name=name,
+                value=value,
+            )
+            wrapped_text = textwrap.fill(
+                full_text,
+                initial_indent='',
+                subsequent_indent='    ',
+                width=self.max_width,
+            )
+            yield wrapped_text + '\n'

stevedore/example/load_as_driver.py

@@ -0,0 +1,37 @@
+from __future__ import print_function
+
+import argparse
+
+from stevedore import driver
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        'format',
+        nargs='?',
+        default='simple',
+        help='the output format',
+    )
+    parser.add_argument(
+        '--width',
+        default=60,
+        type=int,
+        help='maximum output width for text',
+    )
+    parsed_args = parser.parse_args()
+
+    data = {
+        'a': 'A',
+        'b': 'B',
+        'long': 'word ' * 80,
+    }
+
+    mgr = driver.DriverManager(
+        namespace='stevedore.example.formatter',
+        name=parsed_args.format,
+        invoke_on_load=True,
+        invoke_args=(parsed_args.width,),
+    )
+    for chunk in mgr.driver.format(data):
+        print(chunk, end='')

stevedore/example/load_as_extension.py

@@ -0,0 +1,39 @@
+from __future__ import print_function
+
+import argparse
+
+from stevedore import extension
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--width',
+        default=60,
+        type=int,
+        help='maximum output width for text',
+    )
+    parsed_args = parser.parse_args()
+
+    data = {
+        'a': 'A',
+        'b': 'B',
+        'long': 'word ' * 80,
+    }
+
+    mgr = extension.ExtensionManager(
+        namespace='stevedore.example.formatter',
+        invoke_on_load=True,
+        invoke_args=(parsed_args.width,),
+    )
+
+    def format_data(ext, data):
+        return (ext.name, ext.obj.format(data))
+
+    results = mgr.map(format_data, data)
+
+    for name, result in results:
+        print('Formatter: {0}'.format(name))
+        for chunk in result:
+            print(chunk, end='')
+        print('')

stevedore/example/setup.py

@@ -0,0 +1,46 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='stevedore-examples',
+    version='1.0',
+
+    description='Demonstration package for stevedore',
+
+    author='Doug Hellmann',
+    author_email='doug.hellmann@dreamhost.com',
+
+    url='https://github.com/dreamhost/stevedore',
+    download_url='https://github.com/dreamhost/stevedore/tarball/master',
+
+    classifiers=['Development Status :: 3 - Alpha',
+                 'License :: OSI Approved :: Apache Software License',
+                 'Programming Language :: Python',
+                 'Programming Language :: Python :: 2',
+                 'Programming Language :: Python :: 2.7',
+                 'Programming Language :: Python :: 3',
+                 'Programming Language :: Python :: 3.2',
+                 'Programming Language :: Python :: 3.3',
+                 'Intended Audience :: Developers',
+                 'Environment :: Console',
+                 ],
+
+    platforms=['Any'],
+
+    scripts=[],
+
+    provides=['stevedore.examples',
+              ],
+
+    packages=find_packages(),
+    include_package_data=True,
+
+    entry_points={
+        'stevedore.example.formatter': [
+            'simple = stevedore.example.simple:Simple',
+            'field = stevedore.example.fields:FieldList',
+            'plain = stevedore.example.simple:Simple',
+        ],
+    },
+
+    zip_safe=False,
+)

stevedore/example/simple.py

@@ -0,0 +1,20 @@
+from stevedore.example import base
+
+
+class Simple(base.FormatterBase):
+    """A very basic formatter.
+    """
+
+    def format(self, data):
+        """Format the data and return unicode text.
+
+        :param data: A dictionary with string keys and simple types as
+                     values.
+        :type data: dict(str:?)
+        """
+        for name, value in sorted(data.items()):
+            line = '{name} = {value}\n'.format(
+                name=name,
+                value=value,
+            )
+            yield line

stevedore/extension.py

@@ -12,6 +12,11 @@ LOG = logging.getLogger(__name__)
 class Extension(object):
     """Book-keeping object for tracking extensions.
 
+    The arguments passed to the constructor are saved as attributes of
+    the instance using the same names, and can be accessed by the
+    callables passed to :meth:`map` or when iterating over an
+    :class:`ExtensionManager` directly.
+
     :param name: The entry point name.
     :type name: str
     :param entry_point: The EntryPoint instance returned by
@@ -20,6 +25,7 @@ class Extension(object):
     :param plugin: The value returned by entry_point.load()
     :param obj: The object returned by ``plugin(*args, **kwds)`` if the
                 manager invoked the extension on load.
+
     """
 
     def __init__(self, name, entry_point, plugin, obj):

stevedore/tests/test_example_fields.py

@@ -0,0 +1,27 @@
+"""Tests for stevedore.exmaple.fields
+"""
+
+from stevedore.example import fields
+
+
+def test_simple_items():
+    f = fields.FieldList(100)
+    text = ''.join(f.format({'a': 'A', 'b': 'B'}))
+    expected = '\n'.join([
+        ': a : A',
+        ': b : B',
+        '',
+    ])
+    assert text == expected
+
+
+def test_long_item():
+    f = fields.FieldList(25)
+    text = ''.join(f.format({'name': 'a value longer than the allowed width'}))
+    expected = '\n'.join([
+        ': name : a value longer',
+        '    than the allowed',
+        '    width',
+        '',
+    ])
+    assert text == expected

stevedore/tests/test_example_simple.py

@@ -0,0 +1,15 @@
+"""Tests for stevedore.exmaple.simple
+"""
+
+from stevedore.example import simple
+
+
+def test_simple_items():
+    f = simple.Simple(100)
+    text = ''.join(f.format({'a': 'A', 'b': 'B'}))
+    expected = '\n'.join([
+        'a = A',
+        'b = B',
+        '',
+    ])
+    assert text == expected