Add docs

Change-Id: I22ed40b692330ccabe212b75d1d44e54fdd0c97e

doc/Makefile

@@ -0,0 +1,153 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    = -W
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	-rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Nodepool.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Nodepool.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/Nodepool"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Nodepool"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."

doc/source/conf.py

@@ -0,0 +1,235 @@
+# -*- coding: utf-8 -*-
+#
+# Nodepool documentation build configuration file, created by
+# sphinx-quickstart on Fri Jun  8 14:44:26 2012.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('../..'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = [ 'sphinxcontrib.blockdiag', 'sphinxcontrib.programoutput' ]
+#extensions = ['sphinx.ext.intersphinx']
+#intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)}
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Nodepool'
+copyright = u'2014, OpenStack'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Nodepooldoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+  ('index', 'Nodepool.tex', u'Nodepool Documentation',
+   u'OpenStack', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'nodepool', u'Nodepool Documentation',
+     [u'OpenStack'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'Nodepool', u'Nodepool Documentation',
+   u'OpenStack', 'Nodepool', 'One line description of project.',
+   'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'

doc/source/configuration.rst

@@ -0,0 +1,194 @@
+.. _configuration:
+
+Configuration
+=============
+
+Nodepool reads its configuration from ``/etc/nodepool/nodepool.yaml``
+by default.  The configuration file follows the standard YAML syntax
+with a number of sections defined with top level keys.  For example, a
+full configuration file may have the ``providers`` and ``targets``
+sections::
+
+  providers:
+    ...
+  targets:
+    ...
+
+The following sections are available.  All are required unless
+otherwise indicated.
+
+script-dir
+----------
+When creating an image to use when launching new nodes, Nodepool will
+run a script that is expected to prepare the machine before the
+snapshot image is created.  The ``script-dir`` parameter indicates a
+directory that holds all of the scripts needed to accomplish this.
+Nodepool will copy the entire directory to the machine before invoking
+the appropriate script for the image being created.
+
+Example::
+
+  script-dir: /path/to/script/dir
+
+dburi
+-----
+Indicates the URI for the database connection.  See the `SQLAlchemy
+documentation
+<http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>`_
+for the syntax.  Example::
+
+  dburi: 'mysql://nodepool@localhost/nodepool'
+
+cron
+----
+This section is optional.
+
+Nodepool runs several periodic tasks.  The ``image-update`` task
+creates a new image for each of the defined images, typically used to
+keep the data cached on the images up to date.  The ``cleanup`` task
+deletes old images and servers which may have encountered errors
+during their initial deletion.  The ``check`` task attempts to log
+into each node that is waiting to be used to make sure that it is
+still operational.  The following illustrates how to change the
+schedule for these tasks and also indicates their default values::
+
+  cron:
+    image-update: '14 2 * * *'
+    cleanup: '27 */6 * * *'
+    check: '*/15 * * * *'
+
+zmq-publishers
+--------------
+Lists the ZeroMQ endpoints for the Jenkins masters.  Nodepool uses
+this to receive real-time notification that jobs are running on nodes
+or are complete and nodes may be deleted.  Example::
+
+  zmq-publishers:
+    - tcp://jenkins1.example.com:8888
+    - tcp://jenkins2.example.com:8888
+
+gearman-servers
+---------------
+Lists the Zuul Gearman servers that should be consulted for real-time
+demand.  Nodepool will use information from these servers to determine
+if additional nodes should be created to satisfy current demand.
+Example::
+
+  gearman-servers:
+    - host: zuul.example.com
+      port: 4730
+
+The ``port`` key is optional.
+
+providers
+---------
+
+Lists the OpenStack cloud providers Nodepool should use.  Within each
+provider, the Nodepool image types are also defined.  If the resulting
+images from different providers should be equivalent, give them the
+same name.  Example::
+
+  providers:
+    - name: provider1
+      username: 'username'
+      password: 'password'
+      auth-url: 'http://auth.provider1.example.com/'
+      project-id: 'project'
+      service-type: 'compute'
+      service-name: 'compute'
+      region-name: 'region1'
+      max-servers: 96
+      rate: 1.0
+      images:
+        - name: precise
+          base-image: 'Precise'
+          min-ram: 8192
+          setup: prepare_node.sh
+          reset: reset_node.sh
+          username: jenkins
+          private-key: /var/lib/jenkins/.ssh/id_rsa
+        - name: quantal
+          base-image: 'Quantal'
+          min-ram: 8192
+          setup: prepare_node.sh
+          reset: reset_node.sh
+          username: jenkins
+          private-key: /var/lib/jenkins/.ssh/id_rsa
+    - name: provider2
+      username: 'username'
+      password: 'password'
+      auth-url: 'http://auth.provider2.example.com/'
+      project-id: 'project'
+      service-type: 'compute'
+      service-name: 'compute'
+      region-name: 'region1'
+      max-servers: 96
+      rate: 1.0
+      images:
+        - name: precise
+          base-image: 'Fake Precise'
+          min-ram: 8192
+          setup: prepare_node.sh
+          reset: reset_node.sh
+          username: jenkins
+          private-key: /var/lib/jenkins/.ssh/id_rsa
+
+For providers, the `name`, `username`, `password`, `auth-url`,
+`project-id`, and `max-servers` keys are required.  For images, the
+`name`, `base-image`, and `min-ram` keys are required.  The `username`
+and `private-key` values default to the values indicated.  Nodepool
+expects that user to exist after running the script indicated by
+`setup`.
+
+targets
+-------
+
+Lists the Jenkins masters to which Nodepool should attach nodes after
+they are created.  Within each target, the images that are used to
+create nodes for that target are listed (so different targets may
+receive nodes based on either the same or different images).
+Example::
+
+  targets:
+    - name: jenkins1
+      jenkins:
+        url: https://jenkins1.example.org/
+        user: username
+        apikey: key
+        credentials-id: id
+      images:
+        - name: precise
+          providers:
+            - name: provider1
+              min-ready: 2
+            - name: provider2
+              min-ready: 2
+        - name: quantal
+          providers:
+            - name: provider1
+              min-ready: 4
+    - name: jenkins2
+      jenkins:
+        url: https://jenkins2.example.org/
+        user: username
+        apikey: key
+        credentials-id: id
+      images:
+        - name: precise
+          min-ready: 4
+          providers:
+            - name: provider1
+
+For targets, the `name` is required.  If using Jenkins, the `url`,
+`user`, and `apikey` keys are required.  If the `credentials-id` key
+is provided, Nodepool will configure the Jenkins slave to use the
+Jenkins credential identified by that ID, otherwise it will use the
+username and ssh keys configured in the image.
+
+For images specified for a target, all indicated keys are required.
+The name of an image should refer to one of the images specified in
+the `provider` section.  Within the image section, a list of providers
+should be provided; this indicates which providers should be used to
+supply this image to this target.  The `min-ready` field indicates
+that Nodepool should try to keep that number of nodes of this image
+type ready on this target at all times.

doc/source/index.rst

@@ -0,0 +1,30 @@
+.. Nodepool documentation master file, created by
+   sphinx-quickstart on Fri Jun  8 14:44:26 2012.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Nodepool
+========
+
+Nodepool is a system for launching single-use test nodes on demand
+based on images built with cached data.  It is designed to work with
+any OpenStack based cloud, and is part of a suite of tools that form a
+comprehensive test system including Jenkins and Zuul.
+
+Contents:
+
+.. toctree::
+   :maxdepth: 2
+
+   installation
+   configuration
+   scripts
+   operation
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

doc/source/installation.rst

@@ -0,0 +1,67 @@
+:title: Installation
+
+Installation
+============
+
+Nodepool consists of a long-running daemon which uses an SQL database
+and communicates with Jenkins using ZeroMQ.
+
+External Requirements
+---------------------
+
+Jenkins
+~~~~~~~
+
+You should have a Jenkins server running with the `ZMQ Event Publisher
+<http://git.openstack.org/cgit/openstack-infra/zmq-event-publisher/tree/README>`
+plugin installed (it is available in the Jenkins Update Center).  Be
+sure that the machine where you plan to run Nodepool can connect to
+the ZMQ port specified by the plugin on your Jenkins master(s).
+
+Zuul
+~~~~
+
+If you plan to use Nodepool with Zuul (it is optional), you should
+ensure that Nodepool can connect to the gearman port on your Zuul
+server (TCP 4730 by default).  This will allow Nodepool to respond to
+current Zuul demand.  If you elect not to connect Nodepool to Zuul, it
+will still operate in a node-replacement mode.
+
+Database
+~~~~~~~~
+
+Nodepool requires an SQL server.  MySQL with the InnoDB storage engine
+is tested and recommended.  PostgreSQL should work fine.  Due to the
+high number of concurrent connections from Nodepool, SQLite is not
+recommended.  When adding or deleting nodes, Nodepool will hold open a
+database connection for each node.  Be sure to configure the database
+server to support at least a number of connections equal to twice the
+number of nodes you expect to be in use at once.
+
+Statsd and Graphite
+~~~~~~~~~~~~~~~~~~~
+
+If you have a Graphite system with statsd, Nodepool can be configured
+to send information to statsd.
+
+Install Nodepool
+----------------
+
+You may install Nodepool directly from PyPI with pip::
+
+  pip install nodepool
+
+Or install directly from a git checkout with::
+
+  pip install .
+
+Configuration
+-------------
+
+Nodepool has a single required configuration file and an optional
+logging configuration file.
+
+The logging configuration file is in the standard python logging
+`configuration file format
+<http://docs.python.org/2/library/logging.config.html#configuration-file-format>`.
+The Nodepool configuration file is described in configuration_.

doc/source/scripts.rst

@@ -0,0 +1,10 @@
+.. _scripts:
+
+Node Prep Scritps
+=================
+
+Nodepool requires the specification of a script directory
+(`script-dir`) in its configuration.  When Nodepool starts a virtual
+machine for the purpose of creating a snapshot image, all of the files
+within this directory will be copied to the virtual machine so they
+are available for use by the setup script.