From cdd000764ec3b303f6dd35a5a85030aa15514768 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Fri, 8 Jun 2012 19:17:28 -0700 Subject: [PATCH] Add documentation. Change-Id: I8197ec2e52596fa4136f8af9aa93ea06e56d4d0d --- .gitignore | 3 + doc/Makefile | 153 +++++++++++++++++++ doc/source/conf.py | 244 ++++++++++++++++++++++++++++++ doc/source/gating.rst | 96 ++++++++++++ doc/source/index.rst | 32 ++++ doc/source/launchers.rst | 105 +++++++++++++ doc/source/triggers.rst | 37 +++++ doc/source/zuul.rst | 319 +++++++++++++++++++++++++++++++++++++++ etc/layout.yaml-sample | 2 +- setup.cfg | 4 + tox.ini | 11 +- 11 files changed, 1004 insertions(+), 2 deletions(-) create mode 100644 doc/Makefile create mode 100644 doc/source/conf.py create mode 100644 doc/source/gating.rst create mode 100644 doc/source/index.rst create mode 100644 doc/source/launchers.rst create mode 100644 doc/source/triggers.rst create mode 100644 doc/source/zuul.rst create mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore index 289029033a..d301ce7eef 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ zuul.egg-info MANIFEST .tox *.pyc +doc/build +doc/source/sourcecode +*~ diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000000..5014008d0b --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,153 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Zuul.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Zuul.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/Zuul" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Zuul" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000000..8b3c086663 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +# +# Zuul documentation build configuration file, created by +# sphinx-quickstart on Fri Jun 8 14:44:26 2012. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.intersphinx'] + +intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Zuul' +copyright = u'2012, OpenStack' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '1.0' +# The full version, including alpha/beta/rc tags. +release = '1.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Zuuldoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'Zuul.tex', u'Zuul Documentation', + u'OpenStack', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'zuul', u'Zuul Documentation', + [u'OpenStack'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'Zuul', u'Zuul Documentation', + u'OpenStack', 'Zuul', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' diff --git a/doc/source/gating.rst b/doc/source/gating.rst new file mode 100644 index 0000000000..bf245dd272 --- /dev/null +++ b/doc/source/gating.rst @@ -0,0 +1,96 @@ +:title: Project Gating + +Project Gating +============== + +Traditionally, many software development projects merge changes from +developers into the repository, and then identify regressions +resulting from those changes (perhaps by running a test suite with a +continuous integration system such as Jenkins), followed by more +patches to fix those bugs. When the mainline of development is +broken, it can be very frustrating for developers and can cause lost +productivity, particularly so when the number of contributors or +contributions is large. + +The process of gating attempts to prevent changes that introduce +regressions from being merged. This keeps the mainline of development +open and working for all developers, and only when a change is +confirmed to work without disruption is it merged. + +Many projects practice an informal method of gating where developers +with mainline commit access ensure that a test suite runs before +merging a change. With more developers, more changes, and more +comprehensive test suites, that process does not scale very well, and +is not the best use of a developer's time. Zuul can help automate +this process, with a particular emphasis on ensuring large numbers of +changes are tested correctly. + +Zuul was designed to handle the workflow of the OpenStack project, but +can be used with any project. + +A particular focus of Zuul is ensuring correctly ordered testing of +changes in parallel. A gating system should always test each change +applied to the tip of the branch exactly as it is going to be merged. +A simple way to do that would be to test one change at a time, and +merge it only if it passes tests. That works very well, but if +changes take a long time to test, developers may have to wait a long +time for their changes to make it into the repository. With some +projects, it may take hours to test changes, and it is easy for +developers to create changes at a rate faster than they can be tested +and merged. + +Zuul's DependentQueueManager allows for parallel execution of test +jobs for gating while ensuring changes are tested correctly, exactly +as if they had been tested one at a time. It does this by performing +speculative execution of test jobs; it assumes that all jobs will +succeed and tests them in parallel accordingly. If they do succeed, +they can all be merged. However, if one fails, then changes that were +expecting it to succeed are re-tested without the failed change. In +the best case, as many changes as execution contexts are available may +be tested in parallel and merged at once. In the worst case, changes +are tested one at a time (as each subsequent change fails, changes +behind it start again). In practice, the OpenStack project observes +something closer to the best case. + +For example, if a core developer approves five changes in rapid +succession:: + + A, B, C, D, E + +Zuul queues those changes in the order they were approved, and notes +that each subsequent change depends on the one ahead of it merging:: + + A <-- B <-- C <-- D <-- E + +Zuul then starts immediately testing all of the changes in parallel. +But in the case of changes that depend on others, it instructs the +test system to include the changes ahead of it, with the assumption +they pass. That means jobs testing change *B* include change *A* as +well:: + + Jobs for A: merge change A, then test + Jobs for B: merge changes A and B, then test + Jobs for C: merge changes A, B and C, then test + Jobs for D: merge changes A, B, C and D, then test + Jobs for E: merge changes A, B, C, D and E, then test + +If changes *A* and *B* pass tests, and *C*, *D*, and *E* fail:: + + A[pass] <-- B[pass] <-- C[fail] <-- D[fail] <-- E[fail] + +Zuul will merge change *A* followed by change *B*, leaving this queue:: + + C[fail] <-- D[fail] <-- E[fail] + +Since *D* was dependent on *C*, it is not clear whether *D*'s failure is the +result of a defect in *D* or *C*:: + + C[fail] <-- D[unknown] <-- E[unknown] + +Since *C* failed, it will report the failure and drop *C* from the queue:: + + D[unknown] <-- E[unknown] + +This queue is the same as if two new changes had just arrived, so Zuul +starts the process again testing *D* against the tip of the branch, and +*E* against *D*. diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000000..f901fd08f2 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,32 @@ +.. Zuul documentation master file, created by + sphinx-quickstart on Fri Jun 8 14:44:26 2012. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Zuul - A Project Gating System +============================== + +Zuul is a program that is used to gate the source code repository of a +project so that changes are only merged if they pass tests. + +The main component of Zuul is the scheduler. It receives events +related to proposed changes (currently from Gerrit), triggers tests +based on those events (currently on Jenkins), and reports back. + +Contents: + +.. toctree:: + :maxdepth: 2 + + gating + triggers + launchers + zuul + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/doc/source/launchers.rst b/doc/source/launchers.rst new file mode 100644 index 0000000000..d936d4ef98 --- /dev/null +++ b/doc/source/launchers.rst @@ -0,0 +1,105 @@ +:title: Launchers + +Launchers +========= + +Zuul has a modular architecture for launching jobs. Currently only +Jenkins is supported, but it should be fairly easy to add a module to +support other systems. Zuul makes very few assumptions about the +interface to a launcher -- if it can trigger jobs, cancel them, and +receive success or failure reports, it should be able to be used with +Zuul. Patches to this effect are welcome. + +Jenkins +------- + +Zuul works with Jenkins using the Jenkins API and the notification +module. It uses the Jenkins API to trigger jobs, passing in +parameters indicating what should be tested. It recieves +notifications on job completion via the notification API (so jobs must +be conifigured to notify Zuul). + +Jenkins Configuration +~~~~~~~~~~~~~~~~~~~~~ + +Zuul will need access to a Jenkins user. Create a user in Jenkins, +and then visit the configuration page for the user: + + https://jenkins.example.com/user/USERNAME/configure + +And click **Show API Token** to retrieve the API token for that user. +You will need this later when configuring Zuul. Make sure that this +user has appropriate permission to build any jobs that you want Zuul +to trigger. + +Make sure the notification plugin is installed. Visit the plugin +manager on your jenkins: + + https://jenkins.example.com/pluginManager/ + +And install **Jenkins Notification plugin**. The homepage for the +plugin is at: + + https://wiki.jenkins-ci.org/display/JENKINS/Notification+Plugin + +Jenkins Job Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~ + +For each job that you want Zuul to trigger, you will need to add a +notification endpoint for the job on that job's configuration page. +Click **Add Endpoint** and enter the following values: + +**Protocol** + ``HTTP`` +**URL** + ``http://127.0.0.1:8001/jenkins_endpoint`` + +If you are running Zuul on a different server than Jenkins, enter the +appropriate URL. Note that Zuul itself has no access controls, so +ensure that only Jenkins is permitted to access that URL. + +Zuul will pass some parameters to Jenkins for every job it launches. +Check **This build is parameterized**, and add the following fields +with the type **String Parameter**: + +**UUID** + Zuul provided key to link builds with Gerrit events +**GERRIT_PROJECT** + Zuul provided project name +**GERRIT_BRANCH** + Zuul provided branch name +**GERRIT_CHANGES** + Zuul provided list of dependent changes to merge + +You may find it useful to use the ``GERRIT_*`` variables in your job. +In particular, ``GERRIT_CHANGES`` indicates the change or changes that +should be tested. If Zuul has decided that more than one change +should be merged and tested together, they will all be listed in +``GERRIT_CHANGES``. The format for the description of one change is:: + + project:branch:refspec + +And multiple changes are separated by a carat ("^"). E.g.:: + + testproject:master:refs/changes/20/420/1^testproject:master:refs/changes/21/421/1" + +The OpenStack project uses the following script to update the +repository in a workspace and merge appropriate changes: + + https://github.com/openstack/openstack-ci-puppet/blob/master/modules/jenkins_slave/files/slave_scripts/gerrit-git-prep.sh + +Gerrit events that do not include a change (e.g., ref-updated events +which are emitted after a git ref is updated (i.e., a commit is merged +to master)) require a slightly different set of parameters: + +**UUID** + Zuul provided key to link builds with Gerrit events +**GERRIT_PROJECT** + Zuul provided project name +**GERRIT_REFNAME** + Zuul provided ref name +**GERRIT_OLDREV** + Zuul provided old reference for ref-updated +**GERRIT_NEWREV** + Zuul provided new reference for ref-updated + diff --git a/doc/source/triggers.rst b/doc/source/triggers.rst new file mode 100644 index 0000000000..edd98646d7 --- /dev/null +++ b/doc/source/triggers.rst @@ -0,0 +1,37 @@ +:title: Triggers + +Triggers +======== + +The process of merging a change starts with proposing a change to be +merged. Currently Zuul only supports Gerrit as a triggering system. +Zuul's design is modular, so alternate triggering and reporting +systems can be supported. However, Gerrit has a particularly robust +data model, and Zuul does make some assumptions that include that data +model. Nonetheless, patches to support alternate systems are welcome. + +Gerrit +------ + +Zuul works with standard versions of Gerrit by invoking the ``gerrit +stream-events`` command over an SSH connection. It also reports back +to Gerrit using SSH. + +Gerrit Configuration +~~~~~~~~~~~~~~~~~~~~ + +Zuul will need access to a Gerrit user. Consider naming the user +*Jenkins* so that developers see that feedback from changes is from +Jenkins (Zuul attempts to stay out of the way of developers, most +shouldn't even need to know it's there). + +Create an SSH keypair for Zuul to use if there isn't one already, and +create a Gerrit user with that key:: + + cat ~/id_rsa.pub | ssh -p29418 gerrit.example.com gerrit create-account --ssh-key - --full-name Jenkins jenkins + +Give that user whatever permissions will be needed on the projects you +want Zuul to gate. For instance, you may want to grant ``Verified ++/-1`` and ``Submit`` to the user. Additional categories or values may +be added to Gerrit. Zuul is very flexible and can take advantage of +those. diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst new file mode 100644 index 0000000000..9ed4558b5c --- /dev/null +++ b/doc/source/zuul.rst @@ -0,0 +1,319 @@ +:title: Zuul + +Zuul +==== + +Configuration +------------- + +Zuul has three configuration files: + +**zuul.conf** + Credentials for Gerrit and Jenkins, locations of the other config files +**layout.yaml** + Project and queue configuration -- what Zuul does +**logging.conf** + Python logging config + +Examples of each of the three files can be found in the etc/ directory +of the source distribution. + +zuul.conf +~~~~~~~~~ + +Zuul will look for ``/etc/zuul/zuul.conf`` or ``~/zuul.conf`` to +bootstrap its configuration. Alternately, you may specify ``-c +/path/to/zuul.conf`` on the command line. + +Gerrit and Jenkins credentials are each described in a section of +zuul.conf. The location of the other two configuration files (as well +as the location of the PID file when running Zuul as a server) are +specified in a third section. + +layout.yaml +~~~~~~~~~~~ + +This is the main configuration file for Zuul, where all of the queues +and projects are defined, what tests should be run, and what actions +Zuul should perform. There are three sections: queues, jobs, and +projects. + +Queues +"""""" + +Zuul can have any number of independent queues. Whenever a matching +Gerrit event is found for a queue, that event is added to the queue, +and the jobs specified for that queue are run. When all jobs +specified for the queue that were triggered by an event are completed, +Zuul reports back to Gerrit the results. + +There are no pre-defined queues in Zuul, rather you can define +whatever queues you need in the layout file. This is a very flexible +system that can accommodate many kinds of workflows. + +Here is a quick example of a queue definition followed by an +explanation of each of the parameters:: + + - name: check + manager: IndependentQueueManager + trigger: + - event: patchset-created + success: + verified: 1 + failure: + verified: -1 + +**name** + This is used later in the project definition to indicate what jobs + should be run for events in the queue. + +**manager** + There are currently two schemes for managing queues: + + *IndependentQueueManager* + Every event in this queue should be treated as independent of + other events in the queue. This is appropriate when the order of + events in the queue doesn't matter because the results of the + actions this queue performs can not affect other events in the + queue. For example, when a change is first uploaded for review, + you may want to run tests on that change to provide early feedback + to reviewers. At the end of the tests, the change is not going to + be merged, so it is safe to run these tests in parallel without + regard to any other changes in the queue. They are independent. + + Another type of queue that is independent is a post-merge queue. + In that case, the changes have already merged, so the results can + not affect any other events in the queue. + + *DependentQueueManager* + The dependent queue manager is designed for gating. It ensures + that every change is tested exactly as it is going to be merged + into the repository. An ideal gating system would test one change + at a time, applied to the tip of the repository, and only if that + change passed tests would it be merged. Then the next change in + line would be tested the same way. In order to achieve parallel + testing of changes, the dependent queue manager performs + speculative execution on changes. It orders changes based on + their entry into the queue. It begins testing all changes in + parallel, assuming that each change ahead in the queue will pass + its tests. If they all succeed, all the changes can be tested and + merged in parallel. If a change near the front of the queue fails + its tests, each change behind it ignores whatever tests have been + completed and are tested again without the change in front. This + way gate tests may run in parallel but still be tested correctly, + exactly as they will appear in the repository when merged. + + One important characteristic of the DependentQueueManager is that + it analyzes the jobs that are triggered by different projects, and + if those projects have jobs in common, it treats those projects as + related, and they share a single virtual queue of changes. Thus, + if there is a job that performs integration testing on two + projects, those two projects will automatically share a virtual + change queue. If a third project does not invoke that job, it + will be part of a separate virtual change queue, and changes to it + will not depend on changes to the first two jobs. + + For more detail on the theory and operation of Zuul's + DependentQueueManager, see: :doc:`gating`. + +**trigger** + This describes what Gerrit events should be placed in the queue. + Triggers are not exclusive -- matching events may be placed in + multiple queues, and they will behave independently in each of the + queues they match. Multiple triggers may be listed. Further + parameters describe the kind of events that match: + + *event* + The event name from gerrit. Examples: ``patchset-created``, + ``comment-added``, ``ref-updated``. This field is treated as a + regular expression. + + *branch* + The branch associated with the event. Example: ``master``. This + field is treated as a regular expression, and multiple branches may + be listed. + + *ref* + On ref-updated events, the branch parameter is not used, instead the + ref is provided. Currently Gerrit has the somewhat idiosyncratic + behavior of specifying bare refs for branch names (e.g., ``master``), + but full ref names for other kinds of refs (e.g., ``refs/tags/foo``). + Zuul matches what you put here exactly against what Gerrit + provides. This field is treated as a regular expression, and + multiple refs may be listed. + + *approval* + This is only used for ``comment-added`` events. It only matches if + the event has a matching approval associated with it. Example: + ``code-review: 2`` matches a ``+2`` vote on the code review category. + Multiple approvals may be listed. + +**success** + Describes what Zuul should do if all the jobs complete successfully. + This section is optional; if it is omitted, Zuul will run jobs and + do nothing on success; it will not even report a message to Gerrit. + If the section is present, it will leave a message on the Gerrit + review. Each additional argument is assumed to be an argument to + ``gerrit review``, with the boolean value of ``true`` simply + indicating that the argument should be present without following it + with a value. For example, ``verified: 1`` becomes ``gerrit + review --verified 1`` and ``submit: true`` becomes ``gerrit review + --submit``. + +**failure** + Uses the same syntax as **success**, but describes what Zuul should + do if at least one job fails. + +Some example queue configurations are included in the sample layout +file. The first is called a *check* queue:: + + - name: check + manager: IndependentQueueManager + trigger: + - event: patchset-created + success: + verified: 1 + failure: + verified: -1 + +This will trigger jobs each time a new patchset (or change) is +uploaded to Gerrit, and report +/-1 values to Gerrit in the +``verified`` review category. :: + + - name: gate + manager: DependentQueueManager + trigger: + - event: comment-added + approval: + - approved: 1 + success: + verified: 2 + submit: true + failure: + verified: -2 + +This will trigger jobs whenever a reviewer leaves a vote of ``1`` in the +``approved`` review category in Gerrit (a non-standard category). +Changes will be tested in such a way as to guarantee that they will be +merged exactly as tested, though that will happen in parallel by +creating a virtual queue of dependent changes and performing +speculative execution of jobs. :: + + - name: post + manager: IndependentQueueManager + trigger: + - event: ref-updated + ref: ^(?!refs/).*$ + +This will trigger jobs whenever a change is merged to a named branch +(e.g., ``master``). No output will be reported to Gerrit. This is +useful for side effects such as creating per-commit tarballs. :: + + - name: silent + manager: IndependentQueueManager + trigger: + - event: patchset-created + +This also triggers jobs when changes are uploaded to Gerrit, but no +results are reported to Gerrit. This is useful for jobs that are in +development and not yet ready to be presented to developers. + +Jobs +"""" + +The jobs section is optional, and can be used to set attributes of +jobs that are independent of their association with a project. For +example, if a job should return a customized message on failure, that +may be specified here. Otherwise, Zuul does not need to be told about +each job as it builds a list from the project specification. + +**name** + The name of the job. This field is treated as a regular expression + and will be applied to each job that matches. + +**failure-message** + The message that should be reported to Gerrit if the job fails + (optional). + +**success-message** + The message that should be reported to Gerrit if the job fails + (optional). + +**branch** + This job should only be run on matching branches. This field is + treated as a regular expression and multiple branches may be + listed. + +Here is an example of setting the failure message for jobs that check +whether a change merges cleanly:: + + - name: ^.*-merge$ + failure-message: This change was unable to be automatically merged + with the current state of the repository. Please rebase your + change and upload a new patchset. + +Projects +"""""""" + +The projects section indicates what jobs should be run in each queue +for events associated with each project. It contains a list of +projects. Here is an example:: + + - name: example/project + check: + - project-merge: + - project-unittest + - project-pep8 + - project-pyflakes + gate: + - project-merge: + - project-unittest + - project-pep8 + - project-pyflakes + post: + - project-publish + +**name** + The name of the project (as known by Gerrit). + +This is followed by a section for each of the queues defined above. +Queues may be omitted if no jobs should run for this project in a +given queue. Within the queue section, the jobs that should be +executed are listed. If a job is entered as a dictionary key, then +jobs contained within that key are only executed if the key job +succeeds. In the above example, project-unittest, project-pep8, and +project-pyflakes are only executed if project-merge succeeds. This +can help avoid running unnecessary jobs. + +.. seealso:: The OpenStack Zuul configuration for a comprehensive example: https://github.com/openstack/openstack-ci-puppet/blob/master/modules/openstack-ci-config/files/zuul/layout.yaml + + +logging.conf +~~~~~~~~~~~~ +This file is optional. If provided, it should be a standard +:mod:`logging.config` module configuration file. If not present, Zuul will +output all log messages of DEBUG level or higher to the console. + +Starting Zuul +------------- + +To start Zuul, run **zuul-server**:: + + usage: zuul-server [-h] [-c CONFIG] [-d] + + Project gating system. + + optional arguments: + -h, --help show this help message and exit + -c CONFIG specify the config file + -d do not run as a daemon + +You may want to use the ``-d`` argument while you are initially setting +up Zuul so you can detect any configuration errors quickly. Under +normal operation, omit ``-d`` and let Zuul run as a daemon. + +If you send signal 1 (SIGHUP) to the zuul-server process, Zuul will +stop executing new jobs, wait until all executing jobs are finished, +reload its configuration, and resume. Any values in any of the +configuration files may be changed, except the location of Zuul's PID +file (a change to that will be ignored until Zuul is restarted). diff --git a/etc/layout.yaml-sample b/etc/layout.yaml-sample index 39e7aab3f6..8edc79462d 100644 --- a/etc/layout.yaml-sample +++ b/etc/layout.yaml-sample @@ -2,7 +2,7 @@ queues: - name: check manager: IndependentQueueManager trigger: - - event: patchset-uploaded + - event: patchset-created success: verified: 1 failure: diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..8a9134b096 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[build_sphinx] +source-dir = doc/source +build-dir = doc/build +all_files = 1 diff --git a/tox.ini b/tox.ini index 7ec660e432..228eab14d0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,19 @@ [tox] envlist = pep8, pyflakes +[tox:jenkins] +downloadcache = ~/cache/pip + [testenv:pep8] deps = pep8 -commands = pep8 --repeat --show-source zuul zuul-server setup.py +commands = pep8 --repeat --show-source --exclude=.venv,.tox,dist,doc . + +[testenv:cover] +setenv = NOSE_WITH_COVERAGE=1 [testenv:pyflakes] deps = pyflakes commands = pyflakes zuul zuul-server setup.py + +[testenv:venv] +commands = {posargs}