Add docker-compose quickstart example

Use docker-compose to get a zuul+nodepool+gerrit up and configured
so that we can write walkthrough documentation which starts with
a running system.

Change-Id: I5819aee1fb9c3af72eb31818da476df074424225
Co-Authored-By: Tobias Henkel <tobias.henkel@bmw.de>

.gitignore

@@ -2,6 +2,7 @@
 *.egg
 *.egg-info
 *.pyc
+*.retry
 .idea
 .mypy_cache
 .test

doc/source/admin/components.rst

@@ -500,6 +500,12 @@ The following sections of ``zuul.conf`` are used by the executor:
 
       SSH private key file to be used when logging into worker nodes.
 
+   .. attr:: default_username
+      :default: zuul
+
+      Username to use when logging into worker nodes, if none is
+      supplied by Nodepool.
+
    .. attr:: winrm_cert_key_file
       :default: ~/.winrm/winrm_client_cert.key
 

doc/source/admin/examples/ansible-Dockerfile

@@ -0,0 +1,5 @@
+FROM centos:7
+
+RUN yum install -y openssh git
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python get-pip.py
+RUN pip install ansible

doc/source/admin/examples/docker-compose.yaml

@@ -0,0 +1,100 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+  gerrit:
+    image: gerritcodereview/gerrit
+    ports:
+      - "8080:8080"
+      - "29418:29418"
+  gerritconfig:
+    build:
+      dockerfile: ansible-Dockerfile
+      context: ./
+      args:
+        http_proxy: "${http_proxy}"
+        https_proxy: "${http_proxy}"
+        no_proxy: "${no_proxy}"
+    environment:
+      no_proxy: "${no_proxy},gerrit"
+    depends_on:
+      - gerrit
+    volumes:
+      - "sshkey:/var/ssh"
+      - "nodessh:/var/node"
+      - "./playbooks/:/var/playbooks/"
+    command: "ansible-playbook /var/playbooks/setup.yaml"
+  zk:
+     image: zookeeper
+  scheduler:
+    depends_on:
+      - gerritconfig
+      - zk
+    environment:
+      no_proxy: "${no_proxy},gerrit"
+    command: "sh -c 'ansible-playbook /var/playbooks/scheduler.yaml; zuul-scheduler -d'"
+    image: zuul/zuul-scheduler
+    volumes:
+      - "./etc_zuul/:/etc/zuul/"
+      - "./playbooks/:/var/playbooks/"
+      - "sshkey:/var/ssh"
+      - /var/lib/zuul
+  web:
+    command: "zuul-web -d"
+    depends_on:
+      - scheduler
+    ports:
+      - "9000:9000"
+    image: zuul/zuul-web
+    volumes:
+      - "./etc_zuul/:/etc/zuul/"
+  executor:
+    privileged: true
+    environment:
+      no_proxy: "${no_proxy},gerrit"
+    command: "sh -c 'ansible-playbook /var/playbooks/executor.yaml; zuul-executor -d'"
+    depends_on:
+      - scheduler
+    image: zuul/zuul-executor
+    volumes:
+      - "./etc_zuul/:/etc/zuul/"
+      - "./playbooks/:/var/playbooks/"
+      - "sshkey:/var/ssh"
+      - "logs:/srv/static/logs"
+  node:
+    build:
+      dockerfile: node-Dockerfile
+      context: ./
+      args:
+        http_proxy: "${http_proxy}"
+        https_proxy: "${http_proxy}"
+        no_proxy: "${no_proxy}"
+    volumes:
+      - "nodessh:/root/.ssh"
+  launcher:
+    command: "nodepool-launcher -d"
+    depends_on:
+      - zk
+    image: zuul/nodepool
+    volumes:
+      - "./etc_nodepool/:/etc/nodepool/"
+    ports:
+      - "8022:8022"
+  logs:
+    build:
+      dockerfile: logs-Dockerfile
+      context: ./
+      args:
+        http_proxy: "${http_proxy}"
+        https_proxy: "${http_proxy}"
+        no_proxy: "${no_proxy}"
+    ports:
+      - "8000:80"
+    volumes:
+      - "logs:/usr/local/apache2/htdocs"
+
+volumes:
+  sshkey:
+  nodessh:
+  logs:

doc/source/admin/examples/etc_nodepool/nodepool.yaml

@@ -0,0 +1,16 @@
+zookeeper-servers:
+  - host: zk
+
+labels:
+  - name: ubuntu-bionic
+
+providers:
+  - name: static-vms
+    driver: static
+    pools:
+      - name: main
+        nodes:
+          - name: node
+            labels: ubuntu-bionic
+            host-key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFjDZ9C89nUVGQ1qZzG/X0onkmcju4kWJ2uTLBdIXsy1"
+            username: root

doc/source/admin/examples/etc_zuul/main.yaml

@@ -0,0 +1,14 @@
+- tenant:
+    name: example-tenant
+    source:
+      gerrit:
+        config-projects:
+          - zuul-config
+        untrusted-projects:
+          - test1
+          - test2
+      zuul-ci.org:
+        untrusted-projects:
+          - zuul-jobs:
+              include:
+                - job

doc/source/admin/examples/etc_zuul/zuul.conf

@@ -0,0 +1,34 @@
+[gearman]
+server=scheduler
+
+[gearman_server]
+start=true
+
+[zookeeper]
+hosts=zk
+
+[scheduler]
+tenant_config=/etc/zuul/main.yaml
+
+[connection "gerrit"]
+name=gerrit
+driver=gerrit
+server=gerrit
+sshkey=/var/ssh/zuul
+user=zuul
+password=secret
+baseurl=http://gerrit:8080
+auth_type=basic
+
+[connection "zuul-ci.org"]
+name=zuul-ci
+driver=git
+baseurl=https://git.zuul-ci.org/
+
+[web]
+listen_address=0.0.0.0
+
+[executor]
+private_key_file=/var/ssh/nodepool
+default_username=root
+trusted_rw_paths=/srv/static/logs

doc/source/admin/examples/httpd.conf

@@ -0,0 +1,547 @@
+#
+# This is the main Apache HTTP server configuration file.  It contains the
+# configuration directives that give the server its instructions.
+# See <URL:http://httpd.apache.org/docs/2.4/> for detailed information.
+# In particular, see 
+# <URL:http://httpd.apache.org/docs/2.4/mod/directives.html>
+# for a discussion of each configuration directive.
+#
+# Do NOT simply read the instructions in here without understanding
+# what they do.  They're here only as hints or reminders.  If you are unsure
+# consult the online docs. You have been warned.  
+#
+# Configuration and logfile names: If the filenames you specify for many
+# of the server's control files begin with "/" (or "drive:/" for Win32), the
+# server will use that explicit path.  If the filenames do *not* begin
+# with "/", the value of ServerRoot is prepended -- so "logs/access_log"
+# with ServerRoot set to "/usr/local/apache2" will be interpreted by the
+# server as "/usr/local/apache2/logs/access_log", whereas "/logs/access_log" 
+# will be interpreted as '/logs/access_log'.
+
+#
+# ServerRoot: The top of the directory tree under which the server's
+# configuration, error, and log files are kept.
+#
+# Do not add a slash at the end of the directory path.  If you point
+# ServerRoot at a non-local disk, be sure to specify a local disk on the
+# Mutex directive, if file-based mutexes are used.  If you wish to share the
+# same ServerRoot for multiple httpd daemons, you will need to change at
+# least PidFile.
+#
+ServerRoot "/usr/local/apache2"
+
+#
+# Mutex: Allows you to set the mutex mechanism and mutex file directory
+# for individual mutexes, or change the global defaults
+#
+# Uncomment and change the directory if mutexes are file-based and the default
+# mutex file directory is not on a local disk or is not appropriate for some
+# other reason.
+#
+# Mutex default:logs
+
+#
+# Listen: Allows you to bind Apache to specific IP addresses and/or
+# ports, instead of the default. See also the <VirtualHost>
+# directive.
+#
+# Change this to Listen on specific IP addresses as shown below to 
+# prevent Apache from glomming onto all bound IP addresses.
+#
+#Listen 12.34.56.78:80
+Listen 80
+
+#
+# Dynamic Shared Object (DSO) Support
+#
+# To be able to use the functionality of a module which was built as a DSO you
+# have to place corresponding `LoadModule' lines at this location so the
+# directives contained in it are actually available _before_ they are used.
+# Statically compiled modules (those listed by `httpd -l') do not need
+# to be loaded here.
+#
+# Example:
+# LoadModule foo_module modules/mod_foo.so
+#
+LoadModule mpm_event_module modules/mod_mpm_event.so
+#LoadModule mpm_prefork_module modules/mod_mpm_prefork.so
+#LoadModule mpm_worker_module modules/mod_mpm_worker.so
+LoadModule authn_file_module modules/mod_authn_file.so
+#LoadModule authn_dbm_module modules/mod_authn_dbm.so
+#LoadModule authn_anon_module modules/mod_authn_anon.so
+#LoadModule authn_dbd_module modules/mod_authn_dbd.so
+#LoadModule authn_socache_module modules/mod_authn_socache.so
+LoadModule authn_core_module modules/mod_authn_core.so
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
+LoadModule authz_user_module modules/mod_authz_user.so
+#LoadModule authz_dbm_module modules/mod_authz_dbm.so
+#LoadModule authz_owner_module modules/mod_authz_owner.so
+#LoadModule authz_dbd_module modules/mod_authz_dbd.so
+LoadModule authz_core_module modules/mod_authz_core.so
+#LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
+#LoadModule authnz_fcgi_module modules/mod_authnz_fcgi.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule auth_basic_module modules/mod_auth_basic.so
+#LoadModule auth_form_module modules/mod_auth_form.so
+#LoadModule auth_digest_module modules/mod_auth_digest.so
+#LoadModule allowmethods_module modules/mod_allowmethods.so
+#LoadModule isapi_module modules/mod_isapi.so
+#LoadModule file_cache_module modules/mod_file_cache.so
+#LoadModule cache_module modules/mod_cache.so
+#LoadModule cache_disk_module modules/mod_cache_disk.so
+#LoadModule cache_socache_module modules/mod_cache_socache.so
+#LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
+#LoadModule socache_dbm_module modules/mod_socache_dbm.so
+#LoadModule socache_memcache_module modules/mod_socache_memcache.so
+#LoadModule watchdog_module modules/mod_watchdog.so
+#LoadModule macro_module modules/mod_macro.so
+#LoadModule dbd_module modules/mod_dbd.so
+#LoadModule bucketeer_module modules/mod_bucketeer.so
+#LoadModule dumpio_module modules/mod_dumpio.so
+#LoadModule echo_module modules/mod_echo.so
+#LoadModule example_hooks_module modules/mod_example_hooks.so
+#LoadModule case_filter_module modules/mod_case_filter.so
+#LoadModule case_filter_in_module modules/mod_case_filter_in.so
+#LoadModule example_ipc_module modules/mod_example_ipc.so
+#LoadModule buffer_module modules/mod_buffer.so
+#LoadModule data_module modules/mod_data.so
+#LoadModule ratelimit_module modules/mod_ratelimit.so
+LoadModule reqtimeout_module modules/mod_reqtimeout.so
+#LoadModule ext_filter_module modules/mod_ext_filter.so
+#LoadModule request_module modules/mod_request.so
+#LoadModule include_module modules/mod_include.so
+LoadModule filter_module modules/mod_filter.so
+#LoadModule reflector_module modules/mod_reflector.so
+#LoadModule substitute_module modules/mod_substitute.so
+#LoadModule sed_module modules/mod_sed.so
+#LoadModule charset_lite_module modules/mod_charset_lite.so
+#LoadModule deflate_module modules/mod_deflate.so
+#LoadModule xml2enc_module modules/mod_xml2enc.so
+#LoadModule proxy_html_module modules/mod_proxy_html.so
+LoadModule mime_module modules/mod_mime.so
+#LoadModule ldap_module modules/mod_ldap.so
+LoadModule log_config_module modules/mod_log_config.so
+#LoadModule log_debug_module modules/mod_log_debug.so
+#LoadModule log_forensic_module modules/mod_log_forensic.so
+#LoadModule logio_module modules/mod_logio.so
+#LoadModule lua_module modules/mod_lua.so
+LoadModule env_module modules/mod_env.so
+#LoadModule mime_magic_module modules/mod_mime_magic.so
+#LoadModule cern_meta_module modules/mod_cern_meta.so
+#LoadModule expires_module modules/mod_expires.so
+LoadModule headers_module modules/mod_headers.so
+#LoadModule ident_module modules/mod_ident.so
+#LoadModule usertrack_module modules/mod_usertrack.so
+#LoadModule unique_id_module modules/mod_unique_id.so
+LoadModule setenvif_module modules/mod_setenvif.so
+LoadModule version_module modules/mod_version.so
+#LoadModule remoteip_module modules/mod_remoteip.so
+#LoadModule proxy_module modules/mod_proxy.so
+#LoadModule proxy_connect_module modules/mod_proxy_connect.so
+#LoadModule proxy_ftp_module modules/mod_proxy_ftp.so
+#LoadModule proxy_http_module modules/mod_proxy_http.so
+#LoadModule proxy_fcgi_module modules/mod_proxy_fcgi.so
+#LoadModule proxy_scgi_module modules/mod_proxy_scgi.so
+#LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
+#LoadModule proxy_fdpass_module modules/mod_proxy_fdpass.so
+#LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so
+#LoadModule proxy_ajp_module modules/mod_proxy_ajp.so
+#LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
+#LoadModule proxy_express_module modules/mod_proxy_express.so
+#LoadModule proxy_hcheck_module modules/mod_proxy_hcheck.so
+#LoadModule session_module modules/mod_session.so
+#LoadModule session_cookie_module modules/mod_session_cookie.so
+#LoadModule session_crypto_module modules/mod_session_crypto.so
+#LoadModule session_dbd_module modules/mod_session_dbd.so
+#LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
+#LoadModule slotmem_plain_module modules/mod_slotmem_plain.so
+#LoadModule ssl_module modules/mod_ssl.so
+#LoadModule optional_hook_export_module modules/mod_optional_hook_export.so
+#LoadModule optional_hook_import_module modules/mod_optional_hook_import.so
+#LoadModule optional_fn_import_module modules/mod_optional_fn_import.so
+#LoadModule optional_fn_export_module modules/mod_optional_fn_export.so
+#LoadModule dialup_module modules/mod_dialup.so
+#LoadModule http2_module modules/mod_http2.so
+#LoadModule proxy_http2_module modules/mod_proxy_http2.so
+#LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so
+#LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so
+#LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so
+#LoadModule lbmethod_heartbeat_module modules/mod_lbmethod_heartbeat.so
+LoadModule unixd_module modules/mod_unixd.so
+#LoadModule heartbeat_module modules/mod_heartbeat.so
+#LoadModule heartmonitor_module modules/mod_heartmonitor.so
+#LoadModule dav_module modules/mod_dav.so
+LoadModule status_module modules/mod_status.so
+LoadModule autoindex_module modules/mod_autoindex.so
+#LoadModule asis_module modules/mod_asis.so
+#LoadModule info_module modules/mod_info.so
+#LoadModule suexec_module modules/mod_suexec.so
+<IfModule !mpm_prefork_module>
+	#LoadModule cgid_module modules/mod_cgid.so
+</IfModule>
+<IfModule mpm_prefork_module>
+	#LoadModule cgi_module modules/mod_cgi.so
+</IfModule>
+#LoadModule dav_fs_module modules/mod_dav_fs.so
+#LoadModule dav_lock_module modules/mod_dav_lock.so
+#LoadModule vhost_alias_module modules/mod_vhost_alias.so
+#LoadModule negotiation_module modules/mod_negotiation.so
+LoadModule dir_module modules/mod_dir.so
+#LoadModule imagemap_module modules/mod_imagemap.so
+#LoadModule actions_module modules/mod_actions.so
+#LoadModule speling_module modules/mod_speling.so
+#LoadModule userdir_module modules/mod_userdir.so
+LoadModule alias_module modules/mod_alias.so
+#LoadModule rewrite_module modules/mod_rewrite.so
+
+<IfModule unixd_module>
+#
+# If you wish httpd to run as a different user or group, you must run
+# httpd as root initially and it will switch.  
+#
+# User/Group: The name (or #number) of the user/group to run httpd as.
+# It is usually good practice to create a dedicated user and group for
+# running httpd, as with most system services.
+#
+User daemon
+Group daemon
+
+</IfModule>
+
+# 'Main' server configuration
+#
+# The directives in this section set up the values used by the 'main'
+# server, which responds to any requests that aren't handled by a
+# <VirtualHost> definition.  These values also provide defaults for
+# any <VirtualHost> containers you may define later in the file.
+#
+# All of these directives may appear inside <VirtualHost> containers,
+# in which case these default settings will be overridden for the
+# virtual host being defined.
+#
+
+#
+# ServerAdmin: Your address, where problems with the server should be
+# e-mailed.  This address appears on some server-generated pages, such
+# as error documents.  e.g. admin@your-domain.com
+#
+ServerAdmin you@example.com
+
+#
+# ServerName gives the name and port that the server uses to identify itself.
+# This can often be determined automatically, but we recommend you specify
+# it explicitly to prevent problems during startup.
+#
+# If your host doesn't have a registered DNS name, enter its IP address here.
+#
+#ServerName www.example.com:80
+
+#
+# Deny access to the entirety of your server's filesystem. You must
+# explicitly permit access to web content directories in other 
+# <Directory> blocks below.
+#
+<Directory />
+    AllowOverride none
+    Require all denied
+</Directory>
+
+#
+# Note that from this point forward you must specifically allow
+# particular features to be enabled - so if something's not working as
+# you might expect, make sure that you have specifically enabled it
+# below.
+#
+
+#
+# DocumentRoot: The directory out of which you will serve your
+# documents. By default, all requests are taken from this directory, but
+# symbolic links and aliases may be used to point to other locations.
+#
+DocumentRoot "/usr/local/apache2/htdocs"
+<Directory "/usr/local/apache2/htdocs">
+    #
+    # Possible values for the Options directive are "None", "All",
+    # or any combination of:
+    #   Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews
+    #
+    # Note that "MultiViews" must be named *explicitly* --- "Options All"
+    # doesn't give it to you.
+    #
+    # The Options directive is both complicated and important.  Please see
+    # http://httpd.apache.org/docs/2.4/mod/core.html#options
+    # for more information.
+    #
+    Options Indexes FollowSymLinks
+
+    #
+    # AllowOverride controls what directives may be placed in .htaccess files.
+    # It can be "All", "None", or any combination of the keywords:
+    #   AllowOverride FileInfo AuthConfig Limit
+    #
+    AllowOverride None
+
+    #
+    # Controls who can get stuff from this server.
+    #
+    Require all granted
+</Directory>
+
+#
+# DirectoryIndex: sets the file that Apache will serve if a directory
+# is requested.
+#
+<IfModule dir_module>
+    DirectoryIndex index.html
+</IfModule>
+
+#
+# The following lines prevent .htaccess and .htpasswd files from being 
+# viewed by Web clients. 
+#
+<Files ".ht*">
+    Require all denied
+</Files>
+
+#
+# ErrorLog: The location of the error log file.
+# If you do not specify an ErrorLog directive within a <VirtualHost>
+# container, error messages relating to that virtual host will be
+# logged here.  If you *do* define an error logfile for a <VirtualHost>
+# container, that host's errors will be logged there and not here.
+#
+ErrorLog /proc/self/fd/2
+
+#
+# LogLevel: Control the number of messages logged to the error_log.
+# Possible values include: debug, info, notice, warn, error, crit,
+# alert, emerg.
+#
+LogLevel warn
+
+<IfModule log_config_module>
+    #
+    # The following directives define some format nicknames for use with
+    # a CustomLog directive (see below).
+    #
+    LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+    LogFormat "%h %l %u %t \"%r\" %>s %b" common
+
+    <IfModule logio_module>
+      # You need to enable mod_logio.c to use %I and %O
+      LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+    </IfModule>
+
+    #
+    # The location and format of the access logfile (Common Logfile Format).
+    # If you do not define any access logfiles within a <VirtualHost>
+    # container, they will be logged here.  Contrariwise, if you *do*
+    # define per-<VirtualHost> access logfiles, transactions will be
+    # logged therein and *not* in this file.
+    #
+    CustomLog /proc/self/fd/1 common
+
+    #
+    # If you prefer a logfile with access, agent, and referer information
+    # (Combined Logfile Format) you can use the following directive.
+    #
+    #CustomLog "logs/access_log" combined
+</IfModule>
+
+<IfModule alias_module>
+    #
+    # Redirect: Allows you to tell clients about documents that used to 
+    # exist in your server's namespace, but do not anymore. The client 
+    # will make a new request for the document at its new location.
+    # Example:
+    # Redirect permanent /foo http://www.example.com/bar
+
+    #
+    # Alias: Maps web paths into filesystem paths and is used to
+    # access content that does not live under the DocumentRoot.
+    # Example:
+    # Alias /webpath /full/filesystem/path
+    #
+    # If you include a trailing / on /webpath then the server will
+    # require it to be present in the URL.  You will also likely
+    # need to provide a <Directory> section to allow access to
+    # the filesystem path.
+
+    #
+    # ScriptAlias: This controls which directories contain server scripts. 
+    # ScriptAliases are essentially the same as Aliases, except that
+    # documents in the target directory are treated as applications and
+    # run by the server when requested rather than as documents sent to the
+    # client.  The same rules about trailing "/" apply to ScriptAlias
+    # directives as to Alias.
+    #
+    ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/"
+
+</IfModule>
+
+<IfModule cgid_module>
+    #
+    # ScriptSock: On threaded servers, designate the path to the UNIX
+    # socket used to communicate with the CGI daemon of mod_cgid.
+    #
+    #Scriptsock cgisock
+</IfModule>
+
+#
+# "/usr/local/apache2/cgi-bin" should be changed to whatever your ScriptAliased
+# CGI directory exists, if you have that configured.
+#
+<Directory "/usr/local/apache2/cgi-bin">
+    AllowOverride None
+    Options None
+    Require all granted
+</Directory>
+
+<IfModule headers_module>
+    #
+    # Avoid passing HTTP_PROXY environment to CGI's on this or any proxied
+    # backend servers which have lingering "httpoxy" defects.
+    # 'Proxy' request header is undefined by the IETF, not listed by IANA
+    #
+    RequestHeader unset Proxy early
+</IfModule>
+
+<IfModule mime_module>
+    #
+    # TypesConfig points to the file containing the list of mappings from
+    # filename extension to MIME-type.
+    #
+    TypesConfig conf/mime.types
+
+    #
+    # AddType allows you to add to or override the MIME configuration
+    # file specified in TypesConfig for specific file types.
+    #
+    #AddType application/x-gzip .tgz
+    #
+    # AddEncoding allows you to have certain browsers uncompress
+    # information on the fly. Note: Not all browsers support this.
+    #
+    AddEncoding x-compress .Z
+    AddEncoding x-gzip .gz .tgz
+    #
+    # If the AddEncoding directives above are commented-out, then you
+    # probably should define those extensions to indicate media types:
+    #
+    #AddType application/x-compress .Z
+    #AddType application/x-gzip .gz .tgz
+
+    #
+    # AddHandler allows you to map certain file extensions to "handlers":
+    # actions unrelated to filetype. These can be either built into the server
+    # or added with the Action directive (see below)
+    #
+    # To use CGI scripts outside of ScriptAliased directories:
+    # (You will also need to add "ExecCGI" to the "Options" directive.)
+    #
+    #AddHandler cgi-script .cgi
+
+    # For type maps (negotiated resources):
+    #AddHandler type-map var
+
+    #
+    # Filters allow you to process content before it is sent to the client.
+    #
+    # To parse .shtml files for server-side includes (SSI):
+    # (You will also need to add "Includes" to the "Options" directive.)
+    #
+    #AddType text/html .shtml
+    #AddOutputFilter INCLUDES .shtml
+</IfModule>
+
+#
+# The mod_mime_magic module allows the server to use various hints from the
+# contents of the file itself to determine its type.  The MIMEMagicFile
+# directive tells the module where the hint definitions are located.
+#
+#MIMEMagicFile conf/magic
+
+#
+# Customizable error responses come in three flavors:
+# 1) plain text 2) local redirects 3) external redirects
+#
+# Some examples:
+#ErrorDocument 500 "The server made a boo boo."
+#ErrorDocument 404 /missing.html
+#ErrorDocument 404 "/cgi-bin/missing_handler.pl"
+#ErrorDocument 402 http://www.example.com/subscription_info.html
+#
+
+#
+# MaxRanges: Maximum number of Ranges in a request before
+# returning the entire resource, or one of the special
+# values 'default', 'none' or 'unlimited'.
+# Default setting is to accept 200 Ranges.
+#MaxRanges unlimited
+
+#
+# EnableMMAP and EnableSendfile: On systems that support it, 
+# memory-mapping or the sendfile syscall may be used to deliver
+# files.  This usually improves server performance, but must
+# be turned off when serving from networked-mounted 
+# filesystems or if support for these functions is otherwise
+# broken on your system.
+# Defaults: EnableMMAP On, EnableSendfile Off
+#
+#EnableMMAP off
+#EnableSendfile on
+
+# Supplemental configuration
+#
+# The configuration files in the conf/extra/ directory can be 
+# included to add extra features or to modify the default configuration of 
+# the server, or you may simply copy their contents here and change as 
+# necessary.
+
+# Server-pool management (MPM specific)
+#Include conf/extra/httpd-mpm.conf
+
+# Multi-language error messages
+#Include conf/extra/httpd-multilang-errordoc.conf
+
+# Fancy directory listings
+#Include conf/extra/httpd-autoindex.conf
+
+# Language settings
+#Include conf/extra/httpd-languages.conf
+
+# User home directories
+#Include conf/extra/httpd-userdir.conf
+
+# Real-time info on requests and configuration
+#Include conf/extra/httpd-info.conf
+
+# Virtual hosts
+#Include conf/extra/httpd-vhosts.conf
+
+# Local access to the Apache HTTP Server Manual
+#Include conf/extra/httpd-manual.conf
+
+# Distributed authoring and versioning (WebDAV)
+#Include conf/extra/httpd-dav.conf
+
+# Various default settings
+#Include conf/extra/httpd-default.conf
+
+# Configure mod_proxy_html to understand HTML4/XHTML1
+<IfModule proxy_html_module>
+Include conf/extra/proxy-html.conf
+</IfModule>
+
+# Secure (SSL/TLS) connections
+#Include conf/extra/httpd-ssl.conf
+#
+# Note: The following must must be present to support
+#       starting without SSL on platforms with no /dev/random equivalent
+#       but a statically compiled-in mod_ssl.
+#
+<IfModule ssl_module>
+SSLRandomSeed startup builtin
+SSLRandomSeed connect builtin
+</IfModule>

doc/source/admin/examples/logs-Dockerfile

@@ -0,0 +1,3 @@
+FROM httpd:alpine
+
+COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf

doc/source/admin/examples/node-Dockerfile

@@ -0,0 +1,4 @@
+FROM rastasheep/ubuntu-sshd
+
+RUN apt-get update && \
+    apt-get -y install python rsync

doc/source/admin/examples/playbooks/executor.yaml

@@ -0,0 +1,15 @@
+# Until https://review.openstack.org/608453 merges, we need to add
+# Gerrit's SSH host keys to the known hosts_file before starting.
+
+- hosts: localhost
+  tasks:
+    - name: wait for gerrit to start
+      wait_for:
+        host: gerrit
+        port: 29418
+    - name: make ssh dir
+      file:
+        state: directory
+        path: /root/.ssh
+    - name: fetch ssh host keys from gerrit
+      shell: ssh-keyscan -p 29418 gerrit > /root/.ssh/known_hosts

doc/source/admin/examples/playbooks/project.config

@@ -0,0 +1,84 @@
+[project]
+	description = Access inherited by all other projects.
+[receive]
+	requireContributorAgreement = false
+	requireSignedOffBy = false
+	requireChangeId = true
+	enableSignedPush = false
+[submit]
+	mergeContent = true
+[capability]
+	administrateServer = group Administrators
+	priority = batch group Non-Interactive Users
+	streamEvents = group Non-Interactive Users
+[access "refs/*"]
+	read = group Administrators
+	read = group Anonymous Users
+[access "refs/for/*"]
+	addPatchSet = group Registered Users
+	forgeAuthor = group Registered Users
+	forgeCommitter = group Registered Users
+[access "refs/for/refs/*"]
+	push = group Registered Users
+	pushMerge = group Registered Users
+[access "refs/heads/*"]
+	create = group Administrators
+	create = group Project Owners
+	forgeAuthor = group Registered Users
+	forgeCommitter = group Administrators
+	forgeCommitter = group Project Owners
+	forgeCommitter = group Registered Users
+	push = +force group Administrators
+	push = group Project Owners
+	label-Code-Review = -2..+2 group Registered Users
+	label-Verified = -2..+2 group Administrators
+	label-Verified = -2..+2 group Non-Interactive Users
+	label-Workflow = -1..+1 group Registered Users
+	submit = group Administrators
+	submit = group Project Owners
+	submit = group Non-Interactive Users
+	editTopicName = +force group Administrators
+	editTopicName = +force group Project Owners
+[access "refs/meta/config"]
+	exclusiveGroupPermissions = read
+	read = group Administrators
+	read = group Project Owners
+	create = group Administrators
+	create = group Project Owners
+	push = group Administrators
+	push = group Project Owners
+	label-Code-Review = -2..+2 group Administrators
+	label-Code-Review = -2..+2 group Project Owners
+	submit = group Administrators
+	submit = group Project Owners
+[access "refs/tags/*"]
+	create = group Administrators
+	create = group Project Owners
+	createTag = group Administrators
+	createTag = group Project Owners
+	createSignedTag = group Administrators
+	createSignedTag = group Project Owners
+[label "Code-Review"]
+	function = MaxWithBlock
+	defaultValue = 0
+	copyMinScore = true
+	copyAllScoresOnTrivialRebase = true
+	value = -2 This shall not be merged
+	value = -1 I would prefer this is not merged as is
+	value =  0 No score
+	value = +1 Looks good to me, but someone else must approve
+	value = +2 Looks good to me, approved
+[label "Verified"]
+	function = MaxWithBlock
+	value = -2 Fails
+	value = -1 Does not seem to work
+	value =  0 No score
+	value = +1 Works for me
+	value = +2 Verified
+	defaultValue = 0
+[label "Workflow"]
+	function = MaxWithBlock
+	value = -1 Work in progress
+	value =  0 Ready for reviews
+	value = +1 Approved
+	defaultValue = 0

doc/source/admin/examples/playbooks/roles/create_project/tasks/main.yaml

@@ -0,0 +1,33 @@
+- block:
+    - name: check if {{ project }} project exists
+      uri:
+        url: http://gerrit:8080/projects/{{ project }}
+  rescue:
+    - name: create temp dir for {{ project }} creation
+      shell: mktemp -d
+      register: project_tmp
+    - name: create {{ project }} project
+      uri:
+        url: http://gerrit:8080/a/projects/{{ project }}
+        method: PUT
+        user: admin
+        password: secret
+        status_code: 201
+    - name: create initial commit
+      shell:
+        executable: /bin/bash
+        chdir: "{{ project_tmp.stdout }}"
+        cmd: |
+          git init .
+          git config user.name "Admin"
+          git config user.email "admin@example.com"
+          cat >.gitreview <<EOF
+          [gerrit]
+          host=localhost
+          port=29418
+          project={{ project }}
+          EOF
+          git add .gitreview
+          git commit -m "Initial commit"
+          git remote add gerrit http://admin:secret@gerrit:8080/{{ project }}
+          git push -f --set-upstream gerrit +HEAD:master

doc/source/admin/examples/playbooks/scheduler.yaml

@@ -0,0 +1,19 @@
+# The Zuul scheduler needs to be able to connect to the remote systems
+# in order to start.
+
+- hosts: localhost
+  tasks:
+    - name: wait for gerrit to start
+      wait_for:
+        host: gerrit
+        port: 29418
+    - name: wait for zuul user to be created
+      uri:
+        url: http://gerrit:8080/a/accounts/zuul/sshkeys
+        method: GET
+        user: admin
+        password: secret
+      register: result
+      until: result.status == 200 and result.redirected == false
+      delay: 1
+      retries: 120

doc/source/admin/examples/playbooks/setup.yaml

@@ -0,0 +1,102 @@
+- hosts: localhost
+  tasks:
+    - name: wait for gerrit to start
+      wait_for:
+        host: gerrit
+        port: 29418
+    - name: generate admin ssh key
+      command: ssh-keygen -f /var/ssh/admin -N ''
+      args:
+        creates: /var/ssh/admin.pub
+    - name: generate zuul ssh key for gerrit
+      command: ssh-keygen -f /var/ssh/zuul -N ''
+      args:
+        creates: /var/ssh/zuul.pub
+    - name: generate zuul ssh key for nodepool
+      command: ssh-keygen -f /var/ssh/nodepool -N ''
+      args:
+        creates: /var/ssh/nodepool.pub
+    - name: add nodepool key to node authorized_keys file
+      command: cp /var/ssh/nodepool.pub /var/node/authorized_keys
+      args:
+        creates: /var/node/authorized_keys
+    - name: get first login screen
+      uri:
+        url: http://gerrit:8080/a/accounts/self/sshkeys
+        method: GET
+        user: admin
+        password: secret
+      ignore_errors: true
+    - name: add admin ssh key to gerrit
+      uri:
+        url: http://gerrit:8080/a/accounts/self/sshkeys
+        method: POST
+        user: admin
+        password: secret
+        body: "{{ lookup('file', '/var/ssh/admin.pub') }}"
+        status_code: 201
+
+    - name: create temp dir for config update
+      shell: mktemp -d
+      register: gerrit_tmp
+    - name: set fact
+      set_fact:
+        all_projects_repo: "{{ gerrit_tmp.stdout }}/All-Projects"
+    - name: checkout all-projects config
+      git:
+        repo: ssh://gerrit:29418/All-Projects/
+        ssh_opts: "-o StrictHostKeyChecking=no -i /var/ssh/admin -l admin"
+        dest: "{{ all_projects_repo }}"
+        refspec: '+refs/meta/config:refs/meta/config'
+        version: refs/meta/config
+    - name: copy all-projects config
+      copy:
+        src: "/var/playbooks/project.config"
+        dest: "{{ all_projects_repo }}/project.config"
+    - name: update all-projects config
+      shell: |
+        git config user.email 'admin@example.com'
+        git commit -a -m 'update config'
+        git push http://admin:secret@gerrit:8080/All-Projects +HEAD:refs/meta/config
+      args:
+        chdir: "{{ all_projects_repo }}"
+        warn: false
+
+    - name: create zuul-config project
+      include_role:
+        name: create_project
+      vars:
+        project: zuul-config
+    - name: create test1 project
+      include_role:
+        name: create_project
+      vars:
+        project: test1
+    - name: create test2 project
+      include_role:
+        name: create_project
+      vars:
+        project: test2
+
+    # The Zuul user is created last because it is an atomic operation
+    # which signals that Gerrit is ready for use and it is safe for
+    # the Zuul scheduler to start.
+    - block:
+        - name: check if zuul user exists
+          uri:
+            url: http://gerrit:8080/accounts/zuul
+      rescue:
+        - name: create zuul gerrit account
+          uri:
+            url: http://gerrit:8080/a/accounts/zuul
+            method: PUT
+            user: admin
+            password: secret
+            status_code: 201
+            body_format: json
+            body:
+              name: Zuul
+              ssh_key: "{{ lookup('file', '/var/ssh/zuul.pub') }}"
+              http_password: secret
+              groups:
+                - "Non-Interactive Users"

doc/source/admin/examples/test1/playbooks/testjob.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  tasks:
+    - debug:
+        msg: Hello world!

doc/source/admin/examples/test1/zuul.yaml

@@ -0,0 +1,11 @@
+- job:
+    name: testjob
+    run: playbooks/testjob.yaml
+
+- project:
+    check:
+      jobs:
+        - testjob
+    gate:
+      jobs:
+        - testjob

doc/source/admin/examples/zuul-config/playbooks/base/post-logs.yaml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - role: upload-logs
+      zuul_log_url: "http://localhost:8000"

doc/source/admin/examples/zuul-config/playbooks/base/post-ssh.yaml

@@ -0,0 +1,3 @@
+- hosts: all
+  roles:
+    - remove-build-sshkey

doc/source/admin/examples/zuul-config/playbooks/base/pre.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - add-build-sshkey
+    - prepare-workspace

doc/source/admin/examples/zuul-config/zuul.d/jobs.yaml

@@ -0,0 +1,7 @@
+- job:
+    name: base
+    parent: null
+    nodeset:
+      nodes:
+        - name: ubuntu-bionic
+          label: ubuntu-bionic

doc/source/admin/examples/zuul-config/zuul.d/jobs2.yaml

@@ -0,0 +1,22 @@
+- job:
+    name: base
+    parent: null
+    description: |
+      The recommended base job.
+
+      All jobs ultimately inherit from this.  It runs a pre-playbook
+      which copies all of the job's prepared git repos on to all of
+      the nodes in the nodeset.
+
+      It also sets a default timeout value (which may be overidden).
+    pre-run: playbooks/base/pre.yaml
+    post-run:
+      - playbooks/base/post-ssh.yaml
+      - playbooks/base/post-logs.yaml
+    roles:
+      - zuul: zuul-jobs
+    timeout: 1800
+    nodeset:
+      nodes:
+        - name: ubuntu-bionic
+          label: ubuntu-bionic

doc/source/admin/examples/zuul-config/zuul.d/pipelines.yaml

@@ -0,0 +1,51 @@
+- pipeline:
+    name: check
+    description: |
+      Newly uploaded patchsets enter this pipeline to receive an
+      initial +/-1 Verified vote.
+    manager: independent
+    require:
+      gerrit:
+        open: True
+        current-patchset: True
+    trigger:
+      gerrit:
+        - event: patchset-created
+        - event: change-restored
+        - event: comment-added
+          comment: (?i)^(Patch Set [0-9]+:)?( [\w\\+-]*)*(\n\n)?\s*recheck
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- pipeline:
+    name: gate
+    description: |
+      Changes that have been approved are enqueued in order in this
+      pipeline, and if they pass tests, will be merged.
+    manager: dependent
+    post-review: True
+    require:
+      gerrit:
+        open: True
+        current-patchset: True
+        approval:
+          - Workflow: 1
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - Workflow: 1
+    start:
+      gerrit:
+        Verified: 0
+    success:
+      gerrit:
+        Verified: 2
+        submit: true
+    failure:
+      gerrit:
+        Verified: -2

doc/source/admin/examples/zuul-config/zuul.d/projects.yaml

@@ -0,0 +1,15 @@
+- project:
+    name: ^.*$
+    check:
+      jobs: []
+    gate:
+      jobs: []
+
+- project:
+    name: zuul-config
+    check:
+      jobs:
+        - noop
+    gate:
+      jobs:
+        - noop

doc/source/admin/index.rst

@@ -11,6 +11,7 @@ provides to in-project configuration.
 .. toctree::
    :maxdepth: 2
 
+   tutorial
    quick-start
    zuul-from-scratch
    installation

doc/source/admin/tutorial.rst

@@ -0,0 +1,464 @@
+Installation and Setup Tutorial
+===============================
+
+Zuul is not like other CI or CD systems.  It is a project gating
+system designed to assist developers in taking a change from proposal
+through deployment.  Zuul can support any number of workflow processes
+and systems, but to help you get started with Zuul, this tutorial will
+walk through setting up a basic gating configuration which protects
+projects from merging broken code.
+
+This tutorial is entirely self-contained and may safely be run on a
+workstation.  The only requirements are a network connection and the
+ability to run Docker containers.  For code review, it provides an
+instance of Gerrit, though the concepts you will learn apply equally
+to GitHub.  Even if you don't ultimately intend to use Gerrit, you are
+encouraged to follow this tutorial to learn how to set up Zuul and
+then consult further documentation to configure your Zuul to interact
+with GitHub.
+
+Start Zuul Containers
+---------------------
+
+Before you start, ensure that some needed packages are installed.
+
+.. code-block:: shell
+
+   # Debian / Ubuntu:
+
+   sudo apt-get install docker-compose git git-review
+
+   # Red Hat / Fedora / SUSE:
+
+   sudo yum install docker-compose git git-review
+
+Clone the Zuul repository:
+
+.. code-block:: shell
+
+   git clone https://git.zuul-ci.org/zuul
+
+Then cd into the directory containing this document, and run
+docker-compose in order to start Zuul, Nodepool and Gerrit.
+
+.. code-block:: shell
+
+   cd zuul/doc/source/admin/examples
+   docker-compose up
+
+All of the services will be started with debug-level logging sent to
+the standard output of the terminal where docker-compose is running.
+You will see a considerable amount of information scroll by, including
+some errors.  Zuul will immediately attempt to connect to Gerrit and
+begin processing, even before Gerrit has fully initialized.  The
+docker composition includes scripts to configure Gerrit and create an
+account for Zuul.  Once this has all completed, the system should
+automatically connect, stabilize and become idle.  When this is
+complete, you will have the following services running:
+
+* Zookeeper
+* Gerrit
+* Nodepool Launcher
+* Zuul Scheduler
+* Zuul Web Server
+* Zuul Executor
+* Apache HTTPD
+
+And a long-running static test node used by Nodepool and Zuul upon
+which to run tests.
+
+The Zuul scheduler is configured to connect to Gerrit via a connection
+named ``gerrit``.  Zuul can interact with as many systems as
+necessary, each such connection is assigned a name for use in the Zuul
+configuration.
+
+Zuul is a multi-tenant application, so that differing needs of
+independent work-groups can be supported from one system.  This
+example configures a single tenant named ``example-tenant``.  Assigned
+to this tenant are three projects: ``zuul-config``, ``test1`` and
+``test2``.  These have already been created in Gerrit and are ready
+for us to begin using.
+
+Add Your Gerrit Account
+-----------------------
+
+Before you can interact with Gerrit, you will need to create an
+account.  The initialization script has already created an account for
+Zuul, but has left the task of creating your own account to you so
+that you can provide your own SSH key.  You may safely use any
+existing SSH key on your workstation, or you may create a new one by
+running ``ssh-keygen``.
+
+Gerrit is configured in a development mode where passwords are not
+required in the web interface and you may become any user in the
+system at any time.
+
+To create your Gerrit account, visit http://localhost:8080 in your
+browser and click `Become`.
+
+.. image:: images/become.png
+   :align: center
+
+Then click `New Account` under `Register`.
+
+.. image:: images/register.png
+   :align: center
+
+Enter your full name and click `Save Changes`.
+
+.. image:: images/name.png
+   :align: center
+
+Enter the username you use to log into your workstation in the
+`Username` field and click `Select Username`.
+
+.. image:: images/username.png
+   :align: center
+
+Copy and paste the contents of ``~/.ssh/id_rsa.pub`` into the SSH key
+field and click `Add`.
+
+.. image:: images/sshkey.png
+   :align: center
+
+Click the `Continue` link at the bottom of the page.
+
+.. image:: images/continue.png
+   :align: center
+
+At this point you have created and logged into your personal account
+in Gerrit and are ready to begin configuring Zuul.
+
+Configure Zuul Pipelines
+------------------------
+
+Zuul recognizes two types of projects: :term:`config
+projects<config-project>` and :term:`untrusted
+projects<untrusted-project>`.  An *untrusted project* is a normal
+project from Zuul's point of view.  In a gating system, it contains
+the software under development and/or most of the job content that
+Zuul will run.  A *config project* is a special project that contains
+the Zuul's configuration.  Because it has access to normally
+restricted features in Zuul, changes to this repository are not
+dynamically evaluated by Zuul.  The security and functionality of the
+rest of the system depends on this repository, so it is best to limit
+what is contained within it to the minimum, and ensure thorough code
+review practices when changes are made.
+
+Zuul has no built-in workflow definitions, so in order for it to do
+anything, you will need to begin by making changes to a *config
+project*.  The initialization script has already created a project
+named ``zuul-config`` which you should now clone onto your workstation:
+
+.. code-block:: shell
+
+  git clone http://localhost:8080/zuul-config
+
+You will find that this repository is empty.  Zuul reads its
+configuration from either a single file or a directory.  In a *Config
+Project* with substantial Zuul configuration, you may find it easiest
+to use the ``zuul.d`` directory for Zuul configuration.  Later, in
+*Untrusted Projects* you will use a single file for in-repo
+configuration.  Make the directory:
+
+.. code-block:: shell
+
+   cd zuul-config
+   mkdir zuul.d
+
+The first type of configuration items we need to add are the Pipelines
+we intend to use.  In Zuul, a Pipeline represents a workflow action.
+It is triggered by some action on a connection.  Projects are able to
+attach jobs to run in that pipeline, and when they complete, the
+results are reported along with actions which may trigger further
+Pipelines.  In a gating system two pipelines are required:
+:term:`check` and :term:`gate`.  In our system, ``check`` will be
+triggered when a patch is uploaded to Gerrit, so that we are able to
+immediately run tests and report whether the change works and is
+therefore able to merge.  The ``gate`` pipeline is triggered when a code
+reviewer approves the change in Gerrit.  It will run test jobs again
+(in case other changes have merged since the change in question was
+uploaded) and if these final tests pass, will automatically merge the
+change.  To configure these pipelines, copy the following file into
+`zuul.d/pipelines.yaml`:
+
+.. literalinclude:: examples/zuul-config/zuul.d/pipelines.yaml
+   :language: yaml
+
+Once we have bootstrapped our initial Zuul configuration, we will want
+to use the gating process on this repository too, so we need to attach
+the ``zuul-config`` repository to the ``check`` and ``gate`` pipelines
+we are about to create.  There are no jobs defined yet, so we must use
+the internally defined ``noop`` job, which always returns success.
+Later on we will be configuring some other projects, and while we will
+be able to dynamically add jobs to their pipelines, those projects
+must first be attached to the pipelines in order for that to work.  In
+our system, we want all of the projects in Gerrit to participate in
+the check and gate pipelines, so we can use a regular expression to
+apply this to all projects.  To configure the ``check`` and ``gate``
+pipelines for ``zuul-config`` to run the ``noop`` job, and add all
+projects to those pipelines (with no jobs), copy the following file
+into ``zuul.d/projects.yaml``:
+
+.. literalinclude:: examples/zuul-config/zuul.d/projects.yaml
+   :language: yaml
+
+Every real job (i.e., all jobs other than ``noop``) must inherit from a
+:term:`base job`, and base jobs may only be defined in a
+:term:`config-project`.  Let's go ahead and add a simple base job that
+we can build on later.  Copy the following into ``zuul.d/jobs.yaml``:
+
+.. literalinclude:: examples/zuul-config/zuul.d/jobs.yaml
+   :language: yaml
+
+Commit the changes and push them up for review:
+
+.. code-block:: shell
+
+   git add zuul.d
+   git commit -m "Add initial Zuul configuration"
+   git review
+
+Because Zuul is currently running with no configuration whatsoever, it
+will ignore this change.  For this initial change which bootstraps the
+entire system, we will need to bypass code review (hopefully for the
+last time).  To do this, you need to switch to the Administrator
+account in Gerrit.  Visit http://localhost:8080 in your browser and
+then:
+
+Click on your name in the top right corner then click `Switch
+Account`.
+
+.. image:: images/switch-example.png
+   :align: center
+
+Click `admin` to log in as the `admin` user.
+
+.. image:: images/become-select.png
+   :align: center
+
+In the top left corner of the page, click `All` and `Open` to see the
+list of open changes, then click on the change you uploaded.
+
+.. image:: images/open-changes.png
+   :align: center
+
+Click `Reply...` at the top center of the change screen.  This will
+open a dialog where you can leave a review message and vote on the
+change.  As the administrator, you have access to vote in all of the
+review categories, even `Verified` which is normally reserved for
+Zuul.  Vote Code-Review: +2, Verified: +2, Workflow: +1, and then
+click `Post` to leave your approval votes.
+
+.. image:: images/review-1001.png
+   :align: center
+
+Once the required votes have been set, the `Submit` button will
+appear; click it.  This will cause the change to be merged
+immediately.  This is normally handled by Zuul, but as the
+administrator you can bypass Zuul to forcibly merge a change.
+
+.. image:: images/submit-1001.png
+   :align: center
+
+Now that the initial configuration has been bootstrapped, you should
+not need to bypass testing and code review again, so switch back to
+the account you created for yourself.  Click on `Administrator` in the
+top right corner then click `Switch Account`.
+
+.. image:: images/switch-admin.png
+   :align: center
+
+Click your username.
+
+.. image:: images/become-select.png
+   :align: center
+
+Test Zuul Pipelines
+-------------------
+
+Zuul is now running with a basic :term:`check` and :term:`gate`
+configuration.  We can now begin adding Zuul configuration to one of
+our :term:`untrusted projects<untrusted-project>`.  Start by cloning
+the `test1` project which was created by the setup script.
+
+.. code-block:: shell
+
+   cd ..
+   git clone http://localhost:8080/test1
+
+Every Zuul job that runs needs a playbook, so let's create a
+sub-directory in the project to hold playbooks:
+
+.. code-block:: shell
+
+   cd test1
+   mkdir playbooks
+
+Start with a simple playbook which just outputs a debug message.  Copy
+the following to ``playbooks/testjob.yaml``:
+
+.. literalinclude:: examples/test1/playbooks/testjob.yaml
+   :language: yaml
+
+Now define a Zuul job which runs that playbook.  Zuul will read its
+configuration from any of ``zuul.d/`` or ``.zuul.d/`` directories, or
+the files ``zuul.yaml`` or ``.zuul.yaml``.  Generally in an *untrusted
+project* which isn't dedicated entirely to Zuul, it's best to put
+Zuul's configuration in a hidden file.  Copy the following to
+``.zuul.yaml`` in the root of the project:
+
+.. literalinclude:: examples/test1/zuul.yaml
+   :language: yaml
+
+Commit the changes and push them up to Gerrit for review:
+
+.. code-block:: shell
+
+   git add .zuul.yaml playbooks
+   git commit -m "Add test Zuul job"
+   git review
+
+Zuul will dynamically evaluate proposed changes to its configuration
+in *untrusted projects* immediately, so shortly after your change is
+uploaded, Zuul will run the new job and report back on the change.
+
+Visit http://localhost:8080/#/dashboard/self and open the change you
+just uploaded.  If the build is complete, Zuul should have left a
+Verified: +1 vote on the change, along with a comment at the bottom.
+Expand the comments and you should see that the job succeeded, but
+there are no logs and no way to see the output, only a `finger` URL
+(which is what Zuul reports when it doesn't know where build logs are
+stored).
+
+.. image:: images/check1-1002.png
+   :align: center
+
+This means everything is working so far, but we need to configure a
+bit more before we have a useful job.
+
+Configure a Base Job
+--------------------
+
+Every Zuul tenant needs at least one base job.  Zuul administrators
+can use a base job to customize Zuul to the local environment.  This
+may include tasks which run both before jobs, such as setting up
+package mirrors or networking configuration, or after jobs, such as
+artifact and log storage.
+
+Zuul doesn't take anything for granted, and even tasks such as copying
+the git repos for the project being tested onto the remote node must
+be explicitly added to a base job (and can therefore be customized as
+needed).  The Zuul in this tutorial is pre-configured to use the `zuul
+jobs`_ repository which is the "standard library" of Zuul jobs and
+roles.  We will make use of it to quickly create a base job which
+performs the necessary set up actions and stores build logs.
+
+.. _zuul jobs: https://zuul-ci.org/docs/zuul-jobs/
+
+Return to the ``zuul-config`` repo that you were working in earlier.
+We're going to add some playbooks to the empty base job we created
+earlier.  Start by creating a directory to store those playbooks:
+
+.. code-block:: shell
+
+   cd ..
+   cd zuul-config
+   mkdir -p playbooks/base
+
+Zuul supports running any number of playbooks before a job (called
+*pre-run* playbooks) or after a job (called *post-run* playbooks).
+We're going to add a single *pre-run* playbook now.  Copy the
+following to ``playbooks/base/pre.yaml``:
+
+.. literalinclude:: examples/zuul-config/playbooks/base/pre.yaml
+   :language: yaml
+
+This playbook does two things; first it creates a new SSH key and adds
+it to all of the hosts in the inventory, and removes the private key
+that Zuul normally uses to log into nodes from the running SSH agent.
+This is just an extra bit of protection which ensures that if Zuul's
+SSH key has access to any important systems, normal Zuul jobs can't
+use it.  The second thing the playbook does is copy the git
+repositories that Zuul has prepared (which may have one or more
+changes being tested) to all of the nodes used in the job.
+
+Next, add a *post-run* playbook to remove the per-build SSH key.  Copy
+the following to ``playbooks/base/post-ssh.yaml``:
+
+.. literalinclude:: examples/zuul-config/playbooks/base/post-ssh.yaml
+   :language: yaml
+
+This is the complement of the `add-build-sshkey` role in the pre-run
+playbook -- it simply removes the per-build ssh key from any remote
+systems.  Zuul always tries to run all of the post-run playbooks
+regardless of whether any previous playbooks have failed.  Because we
+always want log collection to run and we want it to run last, we
+create a second post-run playbook for it.  Copy the following to
+``playbooks/base/post-logs.yaml``:
+
+.. literalinclude:: examples/zuul-config/playbooks/base/post-logs.yaml
+   :language: yaml
+
+This tutorial is running an Apache webserver in a container which will
+serve build logs from a volume that is shared with the Zuul executor.
+That volume is mounted at `/srv/static/logs`, which is the default
+location in the `upload-logs`_ role.  The role also supports copying
+files to a remote server via SCP; see the role documentation for how
+to configure it.  For this simple case, the only option we need to
+provide is the URL where the logs can ultimately be found.
+
+.. note:: Zuul-jobs also contains a `role
+          <https://zuul-ci.org/docs/zuul-jobs/roles.html#role-upload-logs-swift>`_
+          to upload logs to an OpenStack Object Storage (swift)
+          container.  If you create a role to upload logs to another
+          system, please feel free to contribute it to the zuul-jobs
+          repository for others to use.
+
+.. _upload-logs: https://zuul-ci.org/docs/zuul-jobs/roles.html#role-upload-logs
+
+Now that the new playbooks are in place, update the ``base`` job
+definition to include them.  Overwrite ``zuul.d/jobs.yaml`` with the
+following:
+
+.. literalinclude:: examples/zuul-config/zuul.d/jobs2.yaml
+   :language: yaml
+
+Then commit the change and upload it to Gerrit for review:
+
+.. code-block:: shell
+
+   git add playbooks zuul.d/jobs.yaml
+   git commit -m "Update Zuul base job"
+   git review
+
+Visit http://localhost:8080/#/dashboard/self and open the
+``zuul-config`` change you just uploaded.
+
+You should see a Verified +1 vote from Zuul.  Click `Reply` then vote
+Code-Review: +2 and Workflow: +1 then click 'Post'.
+
+.. image:: images/review-1003.png
+   :align: center
+
+Wait a few moments for Zuul to process the event, and then reload the
+page.  The change should have been merged.
+
+Visit http://localhost:8080/#/dashboard/self and return to the
+``test1`` change you uploaded earlier.  Click `Reply` then type
+`recheck` into the text field and click `Post`.
+
+.. image:: images/recheck-1002.png
+   :align: center
+
+This will cause Zuul to re-run the test job we created earlier.  This
+time it will run with the updated base job configuration, and when
+complete, it will report the published log location as a comment on
+the change:
+
+.. image:: images/check2-1002.png
+   :align: center
+
+Follow the link and you will be able to browse the console log for the
+job.  In the middle of the log, you should see the "Hello, world!"
+output from the job's playbook.