Add an internals doc to the documentation

Sometimes having narrative text describing how objects hang together it
handy for knowing what to do.

Change-Id: Ib12af2d993740d82b4f43de74b11ecefd1cd363a

doc/source/conf.py

@@ -25,7 +25,11 @@ sys.path.insert(0, os.path.abspath('../..'))
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = [ 'sphinxcontrib.blockdiag', 'sphinxcontrib.programoutput' ]
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinxcontrib.blockdiag',
+    'sphinxcontrib.programoutput'
+]
 #extensions = ['sphinx.ext.intersphinx']
 #intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)}
 

doc/source/index.rst

@@ -24,6 +24,7 @@ Contents:
    launchers
    statsd
    client
+   internals
 
 Indices and tables
 ==================

doc/source/internals.rst

@@ -0,0 +1,87 @@
+Zuul Internals
+==============
+
+While most people should not need to understand the details of Zuul's internal
+data model, understanding the data model is essential for people writing
+code for Zuul, and might be interesting to advanced users. The model is
+defined in `zuul/model.py`_.
+
+.. _zuul/model.py: http://git.openstack.org/cgit/openstack-infra/zuul/tree/zuul/model.py
+
+Data Model
+----------
+
+It all starts with the :py:class:`~zuul.model.Pipeline`. A Pipeline is the
+basic organizational structure that everything else hangs off.
+
+.. autoclass:: zuul.model.Pipeline
+
+Pipelines have a configured
+:py:class:`Manager <zuul.manager.BasePipelineManager>` which controlls how
+the :py:class:`Change <zuul.model.Changeish>` objects are enqueued and
+processed.
+
+There are currently two,
+:py:class:`~zuul.manager.dependent.DependentPipelineManager` and
+:py:class:`~zuul.manager.independent.IndependentPipelineManager`
+
+.. autoclass:: zuul.manager.BasePipelineManager
+.. autoclass:: zuul.manager.dependent.DependentPipelineManager
+.. autoclass:: zuul.manager.independent.IndependentPipelineManager
+
+A :py:class:`~zuul.model.Pipeline` has one or more
+:py:class:`~zuul.model.ChangeQueue` objects.
+
+.. autoclass:: zuul.model.ChangeQueue
+
+A :py:class:`~zuul.model.Job` represents the definition of what to do. A
+:py:class:`~zuul.model.Build` represents a single run of a
+:py:class:`~zuul.model.Job`. A :py:class:`~zuul.model.JobTree` is used to
+encapsulate the dependencies between one or more :py:class:`~zuul.model.Job`
+objects.
+
+.. autoclass:: zuul.model.Job
+.. autoclass:: zuul.model.JobTree
+.. autoclass:: zuul.model.Build
+
+The :py:class:`~zuul.manager.base.PipelineManager` enqueues each
+:py:class:`Change <zuul.model.Changeish>` into the
+:py:class:`~zuul.model.ChangeQueue` in a :py:class:`~zuul.model.QueueItem`.
+
+.. autoclass:: zuul.model.QueueItem
+
+As the Changes are processed, each :py:class:`~zuul.model.Build` is put into
+a :py:class:`~zuul.model.BuildSet`
+
+.. autoclass:: zuul.model.BuildSet
+
+Changes
+~~~~~~~
+
+.. autoclass:: zuul.model.Changeish
+.. autoclass:: zuul.model.Change
+.. autoclass:: zuul.model.Ref
+
+Filters
+~~~~~~~
+
+.. autoclass:: zuul.model.ChangeishFilter
+.. autoclass:: zuul.model.EventFilter
+
+
+Tenants
+~~~~~~~
+
+An abide is a collection of tenants.
+
+.. autoclass:: zuul.model.UnparsedAbideConfig
+.. autoclass:: zuul.model.UnparsedTenantConfig
+
+Other Global Objects
+~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: zuul.model.Project
+.. autoclass:: zuul.model.Layout
+.. autoclass:: zuul.model.RepoFiles
+.. autoclass:: zuul.model.Worker
+.. autoclass:: zuul.model.TriggerEvent

zuul/manager/__init__.py

@@ -14,7 +14,6 @@ import extras
 import logging
 
 from zuul import exceptions
-import zuul.configloader
 from zuul.model import NullChange
 
 statsd = extras.try_import('statsd.statsd')
@@ -44,6 +43,8 @@ class StaticChangeQueueContextManager(object):
 
 
 class BasePipelineManager(object):
+    """Abstract Base Class for enqueing and processing Changes in a Pipeline"""
+
     log = logging.getLogger("zuul.BasePipelineManager")
 
     def __init__(self, sched, pipeline):
@@ -453,6 +454,8 @@ class BasePipelineManager(object):
             if build_set.unable_to_merge:
                 return None
             # Load layout
+            # Late import to break an import loop
+            import zuul.configloader
             loader = zuul.configloader.ConfigLoader()
             self.log.debug("Load dynamic layout with %s" % build_set.files)
             layout = loader.createDynamicLayout(item.pipeline.layout.tenant,

zuul/manager/dependent.py

@@ -17,6 +17,13 @@ from zuul.manager import BasePipelineManager, StaticChangeQueueContextManager
 
 
 class DependentPipelineManager(BasePipelineManager):
+    """PipelineManager for handling interrelated Changes.
+
+    The DependentPipelineManager puts Changes that share a Pipeline
+    into a shared :py:class:`~zuul.model.ChangeQueue`. It them processes them
+    using the Optmistic Branch Prediction logic with Nearest Non-Failing Item
+    reparenting algorithm for handling errors.
+    """
     log = logging.getLogger("zuul.DependentPipelineManager")
     changes_merge = True
 

zuul/manager/independent.py

@@ -17,6 +17,8 @@ from zuul.manager import BasePipelineManager, DynamicChangeQueueContextManager
 
 
 class IndependentPipelineManager(BasePipelineManager):
+    """PipelineManager that puts every Change into its own ChangeQueue."""
+
     log = logging.getLogger("zuul.IndependentPipelineManager")
     changes_merge = False
 

zuul/model.py

@@ -70,11 +70,18 @@ def normalizeCategory(name):
 class Pipeline(object):
     """A configuration that ties triggers, reporters, managers and sources.
 
-    Source is where changes should come from. It is a named connection to
+    Source
+        Where changes should come from. It is a named connection to
         an external service defined in zuul.conf
-    Trigger is a description of which events should be processed
+
-    Manager is responsible for enqueing and dequeing Changes
+    Trigger
-    Reporters communicate success and failure results somewhere
+        A description of which events should be processed
+
+    Manager
+        Responsible for enqueing and dequeing Changes
+
+    Reporter
+        Communicates success and failure results somewhere
     """
     def __init__(self, name, layout):
         self.name = name
@@ -196,12 +203,13 @@ class Pipeline(object):
 class ChangeQueue(object):
     """A ChangeQueue contains Changes to be processed related projects.
 
-    DependentPipelines have multiple parallel ChangeQueues shared by
+    A Pipeline with a DependentPipelineManager has multiple parallel
-    different projects.  For instance, there may a ChangeQueue shared by
+    ChangeQueues shared by different projects. For instance, there may a
-    interrelated projects foo and bar, and a second queue for independent
+    ChangeQueue shared by interrelated projects foo and bar, and a second queue
-    project baz.
+    for independent project baz.
 
-    IndependentPipelinesManager puts every Change into its own ChangeQueue
+    A Pipeline with an IndependentPipelineManager puts every Change into its
+    own ChangeQueue
 
     The ChangeQueue Window is inspired by TCP windows and controlls how many
     Changes in a given ChangeQueue will be considered active and ready to