Merge "Reorganize docs"
This commit is contained in:
Zuul
Gerrit Code Review
commit
dbed876f6c
|
|
@ -7,6 +7,7 @@ sphinxcontrib-blockdiag>=1.1.0
|
|||
sphinxcontrib-programoutput
|
||||
sphinx-autodoc-typehints
|
||||
sphinxcontrib-openapi>=0.4.0
|
||||
sphinx_rtd_theme
|
||||
reno>=2.8.0 # Apache-2.0
|
||||
zuul-client
|
||||
zuul-sphinx
|
||||
|
|
|
|||
|
|
@ -1,22 +1,86 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 22.0.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
viewBox="0 0 144 144" style="enable-background:new 0 0 144 144;" xml:space="preserve">
|
||||
<style type="text/css">
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<svg
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
version="1.1"
|
||||
id="Layer_1"
|
||||
x="0px"
|
||||
y="0px"
|
||||
viewBox="0 0 1999.9989 533.33301"
|
||||
xml:space="preserve"
|
||||
sodipodi:docname="Zuul_Logo_Full_Horizontal_RGB_DarkBlue.svg"
|
||||
width="1999.9988"
|
||||
height="533.33301"
|
||||
inkscape:version="1.0.2 (e86c870879, 2021-01-15)"><metadata
|
||||
id="metadata1646"><rdf:RDF><cc:Work
|
||||
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
|
||||
id="defs1644" /><sodipodi:namedview
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1"
|
||||
objecttolerance="10"
|
||||
gridtolerance="10"
|
||||
guidetolerance="10"
|
||||
inkscape:pageopacity="0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:window-width="1983"
|
||||
inkscape:window-height="845"
|
||||
id="namedview1642"
|
||||
showgrid="false"
|
||||
fit-margin-top="0"
|
||||
fit-margin-left="0"
|
||||
fit-margin-right="0"
|
||||
fit-margin-bottom="0"
|
||||
inkscape:zoom="0.2545462"
|
||||
inkscape:cx="107.94999"
|
||||
inkscape:cy="28.799999"
|
||||
inkscape:window-x="235"
|
||||
inkscape:window-y="67"
|
||||
inkscape:window-maximized="0"
|
||||
inkscape:current-layer="Layer_1" />
|
||||
<style
|
||||
type="text/css"
|
||||
id="style1625">
|
||||
.st0{fill:#071D49;}
|
||||
</style>
|
||||
<g>
|
||||
<path class="st0" d="M12.8,102.6h118.5L106,58.8v-15l7-9.2H92L72,0L52,34.7H31.1l7,9.2v15L12.8,102.6z M38,96.6H23.1L38,70.8V96.6z
|
||||
M48.4,96.6H44V61.3h4.3V96.6z M48.4,55.3H44V51h4.3V55.3z M69,96.6H54.3V61.3H69V96.6z M89.7,96.6H75V61.3h14.7V96.6z M89.7,55.3
|
||||
H54.3V51h35.3V55.3z M100,96.6h-4.3V61.3h4.3V96.6z M100,55.3h-4.3V51h4.3V55.3z M106,70.8l14.9,25.8H106V70.8z M72,12l13.1,22.7
|
||||
H58.9L72,12z M100.9,40.7l-0.9,1.2V45H44v-3.2l-0.9-1.2H100.9z"/>
|
||||
<polygon class="st0" points="138.2,137.3 125.1,137.3 125.1,114.6 119.1,118.1 119.1,137.3 119.1,139.6 119.1,143.3 141.6,143.3
|
||||
"/>
|
||||
<path class="st0" d="M99.1,131.5L99.1,131.5L99.1,131.5c0,3.6-2.9,6.5-6.5,6.5c-3.6,0-6.5-2.9-6.5-6.5v0h0v-16.9l-6,3.5v13.5v0
|
||||
c0,6.9,5.6,12.5,12.5,12.5c6.9,0,12.5-5.6,12.5-12.5v0v-16.9l-6,3.5V131.5z"/>
|
||||
<path class="st0" d="M60.2,131.5L60.2,131.5L60.2,131.5c0,3.6-2.9,6.5-6.5,6.5c-3.6,0-6.5-2.9-6.5-6.5v0h0v-16.9l-6,3.5v13.5v0
|
||||
c0,6.9,5.6,12.5,12.5,12.5c6.9,0,12.5-5.6,12.5-12.5v0v-16.9l-6,3.5V131.5z"/>
|
||||
<polygon class="st0" points="25.8,114.6 25.4,114.6 18.9,114.6 5.8,114.6 2.4,120.6 15.5,120.6 2.4,143.3 23.8,143.3 27.3,137.3
|
||||
12.7,137.3 "/>
|
||||
<g
|
||||
id="g1639"
|
||||
transform="matrix(9.2592535,0,0,9.2592535,-316.66647,-399.99975)"
|
||||
style="fill:#ffffff;fill-opacity:1">
|
||||
<path
|
||||
class="st0"
|
||||
d="m 34.2,100.8 h 66.5 L 86.5,76.2 v -8.4 l 3.9,-5.1 H 78.6 L 67.4,43.2 56.2,62.7 H 44.4 l 3.9,5.1 v 8.4 z M 48.4,97.4 H 40 l 8.4,-14.5 z m 5.7,0 H 51.7 V 77.6 h 2.4 z m 0,-23.1 h -2.4 v -2.4 h 2.4 z M 65.7,97.4 H 57.5 V 77.6 h 8.2 z m 11.6,0 H 69.1 V 77.6 h 8.2 z m 0,-23.1 H 57.5 v -2.4 h 19.8 z m 5.8,23.1 H 80.7 V 77.6 h 2.4 z m 0,-23.1 h -2.4 v -2.4 h 2.4 z m 3.4,8.6 8.4,14.5 h -8.4 z m -19.1,-33 7.4,12.7 H 60 Z M 83.6,66 83.1,66.7 v 1.8 H 51.7 V 66.7 L 51.2,66 Z"
|
||||
id="path1627"
|
||||
style="fill:#ffffff;fill-opacity:1" />
|
||||
<g
|
||||
id="g1637"
|
||||
style="fill:#ffffff;fill-opacity:1">
|
||||
<polygon
|
||||
class="st0"
|
||||
points="227.5,86 250.2,86 246.7,80 233.5,80 233.5,57.2 227.5,60.7 "
|
||||
id="polygon1629"
|
||||
style="fill:#ffffff;fill-opacity:1" />
|
||||
<path
|
||||
class="st0"
|
||||
d="m 207.5,74.2 v 0 0 c 0,3.6 -2.9,6.5 -6.5,6.5 -3.6,0 -6.5,-2.9 -6.5,-6.5 v 0 0 -17 l -6,3.5 v 13.5 0 c 0,6.9 5.6,12.5 12.5,12.5 6.9,0 12.5,-5.6 12.5,-12.5 v 0 -17 l -6,3.5 z"
|
||||
id="path1631"
|
||||
style="fill:#ffffff;fill-opacity:1" />
|
||||
<path
|
||||
class="st0"
|
||||
d="m 168.4,74.2 v 0 0 c 0,3.6 -2.9,6.5 -6.5,6.5 -3.6,0 -6.5,-2.9 -6.5,-6.5 v 0 0 -17 l -6,3.5 v 13.5 0 c 0,6.9 5.6,12.5 12.5,12.5 6.9,0 12.5,-5.6 12.5,-12.5 v 0 -17 l -6,3.5 z"
|
||||
id="path1633"
|
||||
style="fill:#ffffff;fill-opacity:1" />
|
||||
<polygon
|
||||
class="st0"
|
||||
points="110.4,86 131.9,86 135.4,80 120.8,80 133.9,57.2 113.8,57.2 110.4,63.3 123.5,63.3 "
|
||||
id="polygon1635"
|
||||
style="fill:#ffffff;fill-opacity:1" />
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 1.5 KiB After Width: | Height: | Size: 3.2 KiB |
|
|
@ -0,0 +1,60 @@
|
|||
:title: About Zuul
|
||||
|
||||
.. _about-zuul:
|
||||
|
||||
About Zuul
|
||||
==========
|
||||
|
||||
Zuul is a Project Gating System. That's like a CI or CD system, but
|
||||
the focus is on testing the future state of code repositories.
|
||||
|
||||
A gating system doesn't just test a proposed change; it tests the
|
||||
proposed future state of multiple branches and repositories with any
|
||||
number of in-flight changes and their dependencies. And the same
|
||||
playbooks used to test software can also be used to deploy it.
|
||||
|
||||
Zuul itself is a service which listens to events from various code
|
||||
review systems, executes jobs based on those events, and reports the
|
||||
results back to the code review system. The primary interface for
|
||||
Zuul is the code review system (or systems) so that it fits seamlessly
|
||||
into developer workflows, and a web interface is available for
|
||||
inspecting the current status and browsing build results.
|
||||
|
||||
The best way to run Zuul is with a single installation serving as many
|
||||
projects or groups as possible. It is a multi-tenant application that
|
||||
is able to provide as much or as little separation between projects as
|
||||
desired.
|
||||
|
||||
Zuul works with a wide range of code review systems, and can work with
|
||||
multiple systems (including integrating projects on different systems)
|
||||
simultaneously. See :ref:`drivers` for a complete list.
|
||||
|
||||
Zuul uses a separate component called `Nodepool`_ to provide the
|
||||
resources to run jobs. Nodepool works with several cloud providers
|
||||
as well as statically defined nodes (again, simultaneously).
|
||||
|
||||
Because Zuul is designed from the ground up to run jobs in a
|
||||
multi-node environment (whether those nodes are bare metal machines,
|
||||
VMs, Kubernetes clusters, or containers), Zuul's job definition
|
||||
language needs to support orchestrating tasks on multiple nodes. Zuul
|
||||
uses Ansible for this. Ansible is well-known and easy to learn and
|
||||
use. Some existing Ansible playbooks and roles may be able to be used
|
||||
directly with Zuul (but some restrictions apply, so not all will).
|
||||
|
||||
However, knowledge or use of Ansible is not required for Zuul -- it is
|
||||
quite simple for Zuul's embedded Ansible to run any shell script or
|
||||
any other program. Zuul's library of standard jobs even includes a
|
||||
job that will run a specified shell script, so it's possible to use
|
||||
Zuul without writing any Ansible at all.
|
||||
|
||||
Zuul is an open source project developed and maintained by a community
|
||||
of users. We welcome your `support and contribution
|
||||
<https://zuul-ci.org/community.html>`__.
|
||||
|
||||
.. toctree::
|
||||
:hidden:
|
||||
|
||||
concepts
|
||||
gating
|
||||
|
||||
_`Nodepool`: https://zuul-ci.org/docs/nodepool/
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
Service Administration
|
||||
======================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
installation
|
||||
components
|
||||
configuration
|
||||
drivers/index
|
||||
tenants
|
||||
operation
|
||||
authentication
|
||||
monitoring
|
||||
client
|
||||
troubleshooting
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
:title: Tenant Scoped REST API
|
||||
:title: Authenticated Actions
|
||||
|
||||
.. _tenant-scoped-rest-api:
|
||||
.. _authentication:
|
||||
|
||||
Tenant Scoped REST API
|
||||
======================
|
||||
Authenticated Actions
|
||||
=====================
|
||||
|
||||
Users can perform some privileged actions at the tenant level through protected
|
||||
endpoints of the REST API, if these endpoints are activated.
|
||||
|
|
@ -129,3 +129,26 @@ For example, in Python, and for an authenticator using the ``HS256`` algorithm:
|
|||
|
||||
Online resources like https://jwt.io are also available to generate, decode and
|
||||
debug JWTs.
|
||||
|
||||
Debugging
|
||||
---------
|
||||
|
||||
If problems appear:
|
||||
|
||||
* Make sure your configuration is correct, especially callback URIs.
|
||||
* More information can be found in Zuul's web service logs.
|
||||
* From the user's side, activating the web console in the browser can be helpful
|
||||
to debug API calls.
|
||||
|
||||
Interfacing with Other Systems
|
||||
------------------------------
|
||||
|
||||
Here are some how-tos to help administrators enable OpenID Connect
|
||||
authentication in Zuul and Zuul's Web UI.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
howtos/openid-with-google
|
||||
howtos/openid-with-keycloak
|
||||
tutorials/keycloak
|
||||
|
|
@ -109,7 +109,7 @@ the jobs, pass the failed tag as the ``ref`` argument and set
|
|||
``newrev`` to the change associated with the tag in the project
|
||||
repository (i.e. what you see from ``git show X.Y.Z``)::
|
||||
|
||||
zuul enqueue-ref --tenant openstack --trigger gerrit --pipeline release --project openstack/example_project --ref refs/tags/X.Y.Z --newrev abc123...
|
||||
zuul enqueue-ref --tenant openstack --trigger gerrit --pipeline release --project openstack/example_project --ref refs/tags/X.Y.Z --newrev abc123..
|
||||
|
||||
The command can also be used asynchronosly trigger a job in a
|
||||
``periodic`` pipeline that would usually be run at a specific time by
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
:title: Component Overview
|
||||
|
||||
Component Overview
|
||||
==================
|
||||
|
||||
.. _components:
|
||||
|
||||
Zuul is a distributed system consisting of several components, each of
|
||||
which is described below.
|
||||
|
||||
.. graphviz::
|
||||
:align: center
|
||||
|
||||
graph {
|
||||
node [shape=box]
|
||||
Database [fontcolor=grey]
|
||||
Executor [href="#executor"]
|
||||
Finger [href="#finger-gateway"]
|
||||
Gearman [shape=ellipse]
|
||||
Gerrit [fontcolor=grey]
|
||||
Merger [href="#merger"]
|
||||
Statsd [shape=ellipse fontcolor=grey]
|
||||
Scheduler [href="#scheduler"]
|
||||
Zookeeper [shape=ellipse]
|
||||
Nodepool
|
||||
GitHub [fontcolor=grey]
|
||||
Web [href="#web-server"]
|
||||
|
||||
Merger -- Gearman
|
||||
Executor -- Statsd
|
||||
Executor -- "Job Node"
|
||||
Web -- Database
|
||||
Web -- Gearman
|
||||
Web -- GitHub
|
||||
Web -- Zookeeper
|
||||
Web -- Executor
|
||||
Finger -- Gearman
|
||||
Finger -- Executor
|
||||
|
||||
Gearman -- Scheduler;
|
||||
Scheduler -- Database;
|
||||
Scheduler -- Gerrit;
|
||||
Scheduler -- Zookeeper;
|
||||
Zookeeper -- Executor;
|
||||
Zookeeper -- Finger;
|
||||
Zookeeper -- Merger
|
||||
Zookeeper -- Nodepool;
|
||||
Scheduler -- GitHub;
|
||||
Scheduler -- Statsd;
|
||||
}
|
||||
|
||||
.. contents::
|
||||
:depth: 1
|
||||
:local:
|
||||
:backlinks: none
|
||||
|
||||
Each of the Zuul processes may run on the same host, or different
|
||||
hosts. Within Zuul, the components communicate with the scheduler via
|
||||
the Gearman protocol, so each Zuul component needs to be able to
|
||||
connect to the host running the Gearman server (the scheduler has a
|
||||
built-in Gearman server which is recommended) on the Gearman port --
|
||||
TCP port 4730 by default.
|
||||
|
||||
The Zuul scheduler communicates with Nodepool via the ZooKeeper
|
||||
protocol. Nodepool requires an external ZooKeeper cluster, and the
|
||||
Zuul scheduler needs to be able to connect to the hosts in that
|
||||
cluster on TCP port 2181 or 2281.
|
||||
|
||||
Both the Nodepool launchers and Zuul executors need to be able to
|
||||
communicate with the hosts which nodepool provides. If these are on
|
||||
private networks, the Executors will need to be able to route traffic
|
||||
to them.
|
||||
|
||||
Only Zuul fingergw and Zuul web need to be publicly accessible;
|
||||
executors never do. Executors should be accessible on TCP port 7900
|
||||
by fingergw and web.
|
||||
|
||||
A database is required and configured in ``database`` section of
|
||||
``/etc/zuul/zuul.conf``. Both Zuul scheduler and Zuul web will need
|
||||
access to it.
|
||||
|
||||
If statsd is enabled, the executors and scheduler needs to be able to
|
||||
emit data to statsd. Statsd can be configured to run on each host
|
||||
and forward data, or services may emit to a centralized statsd
|
||||
collector. Statsd listens on UDP port 8125 by default.
|
||||
|
||||
A minimal Zuul system may consist of a :ref:`scheduler` and
|
||||
:ref:`executor` both running on the same host. Larger installations
|
||||
should consider running multiple executors, each on a dedicated host,
|
||||
and running mergers on dedicated hosts as well.
|
||||
|
|
@ -3,14 +3,14 @@
|
|||
Zuul Concepts
|
||||
=============
|
||||
|
||||
Zuul's configuration is organized around the concept of a *pipeline*.
|
||||
In Zuul, a pipeline encompasses a workflow process which can be
|
||||
applied to one or more projects. For instance, a "check" pipeline
|
||||
might describe the actions which should cause newly proposed changes
|
||||
to projects to be tested. A "gate" pipeline might implement
|
||||
:ref:`project_gating` to automate merging changes to projects only if
|
||||
their tests pass. A "post" pipeline might update published
|
||||
documentation for a project when changes land.
|
||||
Zuul is organized around the concept of a *pipeline*. In Zuul, a
|
||||
pipeline encompasses a workflow process which can be applied to one or
|
||||
more projects. For instance, a "check" pipeline might describe the
|
||||
actions which should cause newly proposed changes to projects to be
|
||||
tested. A "gate" pipeline might implement :ref:`project_gating` to
|
||||
automate merging changes to projects only if their tests pass. A
|
||||
"post" pipeline might update published documentation for a project
|
||||
when changes land.
|
||||
|
||||
The names "check", "gate", and "post" are arbitrary -- these are not
|
||||
concepts built into Zuul, but rather they are just a few common
|
||||
|
|
@ -35,6 +35,7 @@ extensions = [
|
|||
'zuul_sphinx',
|
||||
'zuul.sphinx.ansible',
|
||||
'reno.sphinxext',
|
||||
'sphinx_rtd_theme',
|
||||
]
|
||||
#extensions = ['sphinx.ext.intersphinx']
|
||||
#intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)}
|
||||
|
|
@ -58,7 +59,7 @@ master_doc = 'index'
|
|||
|
||||
# General information about the project.
|
||||
project = u'Zuul'
|
||||
copyright = u'2012, OpenStack'
|
||||
copyright = u'2012-2021, Zuul contributors'
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
|
|
@ -100,13 +101,15 @@ pygments_style = 'sphinx'
|
|||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
#html_theme = 'alabaster'
|
||||
html_theme = "sphinx_rtd_theme"
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
# further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
html_theme_options = {
|
||||
'show_related': True,
|
||||
'logo': 'logo.svg',
|
||||
'collapse_navigation': False,
|
||||
'navigation_depth': -1,
|
||||
'logo_only': True,
|
||||
}
|
||||
|
||||
# Add any paths that contain custom themes here, relative to this directory.
|
||||
|
|
@ -121,7 +124,7 @@ html_theme_options = {
|
|||
|
||||
# The name of an image file (relative to this directory) to place at the top
|
||||
# of the sidebar.
|
||||
#html_logo = '_static/logo.svg'
|
||||
html_logo = '_static/logo.svg'
|
||||
|
||||
# The name of an image file (within the static path) to use as favicon of the
|
||||
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ success, the pipeline reports back to Gerrit with ``Verified`` vote of
|
|||
appear in the repository when merged.
|
||||
|
||||
For more detail on the theory and operation of Zuul's
|
||||
dependent pipeline manager, see: :doc:`/discussion/gating`.
|
||||
dependent pipeline manager, see: :doc:`/gating`.
|
||||
|
||||
.. value:: serial
|
||||
|
||||
|
|
@ -1,101 +1,17 @@
|
|||
:title: Components
|
||||
:title: Configuration
|
||||
|
||||
.. _components:
|
||||
|
||||
Components
|
||||
==========
|
||||
|
||||
.. contents::
|
||||
:depth: 1
|
||||
:local:
|
||||
:backlinks: none
|
||||
|
||||
Overview
|
||||
--------
|
||||
|
||||
Zuul is a distributed system consisting of several components, each of
|
||||
which is described below.
|
||||
|
||||
.. graphviz::
|
||||
:align: center
|
||||
|
||||
graph {
|
||||
node [shape=box]
|
||||
Database [fontcolor=grey]
|
||||
Executor [href="#executor"]
|
||||
Finger [href="#finger-gateway"]
|
||||
Gearman [shape=ellipse]
|
||||
Gerrit [fontcolor=grey]
|
||||
Merger [href="#merger"]
|
||||
Statsd [shape=ellipse fontcolor=grey]
|
||||
Scheduler [href="#scheduler"]
|
||||
Zookeeper [shape=ellipse]
|
||||
Nodepool
|
||||
GitHub [fontcolor=grey]
|
||||
Web [href="#web-server"]
|
||||
|
||||
Merger -- Gearman
|
||||
Executor -- Statsd
|
||||
Executor -- "Job Node"
|
||||
Web -- Database
|
||||
Web -- Gearman
|
||||
Web -- GitHub
|
||||
Web -- Zookeeper
|
||||
Web -- Executor
|
||||
Finger -- Gearman
|
||||
Finger -- Executor
|
||||
|
||||
Gearman -- Scheduler;
|
||||
Scheduler -- Database;
|
||||
Scheduler -- Gerrit;
|
||||
Scheduler -- Zookeeper;
|
||||
Zookeeper -- Executor;
|
||||
Zookeeper -- Finger;
|
||||
Zookeeper -- Merger
|
||||
Zookeeper -- Nodepool;
|
||||
Scheduler -- GitHub;
|
||||
Scheduler -- Statsd;
|
||||
}
|
||||
|
||||
Each of the Zuul processes may run on the same host, or different
|
||||
hosts. Within Zuul, the components communicate with the scheduler via
|
||||
the Gearman protocol, so each Zuul component needs to be able to
|
||||
connect to the host running the Gearman server (the scheduler has a
|
||||
built-in Gearman server which is recommended) on the Gearman port --
|
||||
TCP port 4730 by default.
|
||||
|
||||
The Zuul scheduler communicates with Nodepool via the ZooKeeper
|
||||
protocol. Nodepool requires an external ZooKeeper cluster, and the
|
||||
Zuul scheduler needs to be able to connect to the hosts in that
|
||||
cluster on TCP port 2181 or 2281.
|
||||
|
||||
Both the Nodepool launchers and Zuul executors need to be able to
|
||||
communicate with the hosts which nodepool provides. If these are on
|
||||
private networks, the Executors will need to be able to route traffic
|
||||
to them.
|
||||
|
||||
Only Zuul fingergw and Zuul web need to be publicly accessible;
|
||||
executors never do. Executors should be accessible on TCP port 7900
|
||||
by fingergw and web.
|
||||
|
||||
A database is required and configured in ``database`` section of
|
||||
``/etc/zuul/zuul.conf``. Both Zuul scheduler and Zuul web will need
|
||||
access to it.
|
||||
|
||||
If statsd is enabled, the executors and scheduler needs to be able to
|
||||
emit data to statsd. Statsd can be configured to run on each host
|
||||
and forward data, or services may emit to a centralized statsd
|
||||
collector. Statsd listens on UDP port 8125 by default.
|
||||
Configuration
|
||||
=============
|
||||
|
||||
All Zuul processes read the ``/etc/zuul/zuul.conf`` file (an alternate
|
||||
location may be supplied on the command line) which uses an INI file
|
||||
syntax. Each component may have its own configuration file, though
|
||||
you may find it simpler to use the same file for all components.
|
||||
|
||||
Zuul will interpolate environment variables starting with the ``ZUUL_``
|
||||
prefix given in the config file escaped as python string expansion.
|
||||
``foo=%(ZUUL_HOME)s`` will set the value of ``foo`` to the same value
|
||||
as the environment variable named ``ZUUL_HOME``.
|
||||
Zuul will interpolate environment variables starting with the
|
||||
``ZUUL_`` prefix given in the config file escaped as python string
|
||||
expansion. ``foo=%(ZUUL_HOME)s`` will set the value of ``foo`` to the
|
||||
same value as the environment variable named ``ZUUL_HOME``.
|
||||
|
||||
An example ``zuul.conf``:
|
||||
|
||||
|
|
@ -111,6 +27,9 @@ An example ``zuul.conf``:
|
|||
[zookeeper]
|
||||
hosts=zk1.example.com,zk2.example.com,zk3.example.com
|
||||
|
||||
[database]
|
||||
dburi=<URI>
|
||||
|
||||
[keystore]
|
||||
password=MY_SECRET_PASSWORD
|
||||
|
||||
|
|
@ -120,30 +39,13 @@ An example ``zuul.conf``:
|
|||
[scheduler]
|
||||
log_config=/etc/zuul/scheduler-logging.yaml
|
||||
|
||||
A minimal Zuul system may consist of a :ref:`scheduler` and
|
||||
:ref:`executor` both running on the same host. Larger installations
|
||||
should consider running multiple executors, each on a dedicated host,
|
||||
and running mergers on dedicated hosts as well.
|
||||
|
||||
Zuul stores private keys for each project it knows about in ZooKeeper.
|
||||
It is recommended that you periodically back up the private keys in
|
||||
case the ZooKeeper data store is lost or damaged. The :title:`Zuul
|
||||
Client` provides two sub-commands for use in this case:
|
||||
:title:`export-keys` and :title:`import-keys`. Each takes an argument to
|
||||
a filesystem path and will write the keys to, or read the keys from
|
||||
that path. The data in the exported files are still secured with the
|
||||
keystore passphrase, so be sure to retain it as well.
|
||||
|
||||
Common
|
||||
------
|
||||
|
||||
The following applies to all Zuul components.
|
||||
|
||||
Configuration
|
||||
~~~~~~~~~~~~~
|
||||
Common Options
|
||||
--------------
|
||||
|
||||
The following sections of ``zuul.conf`` are used by all Zuul components:
|
||||
|
||||
Gearman
|
||||
~~~~~~~
|
||||
|
||||
.. attr:: gearman
|
||||
|
||||
|
|
@ -172,6 +74,10 @@ The following sections of ``zuul.conf`` are used by all Zuul components:
|
|||
|
||||
An openssl file containing the client private key in PEM format.
|
||||
|
||||
|
||||
Statsd
|
||||
~~~~~~
|
||||
|
||||
.. attr:: statsd
|
||||
|
||||
Information about the optional statsd server. If the ``statsd``
|
||||
|
|
@ -193,6 +99,9 @@ The following sections of ``zuul.conf`` are used by all Zuul components:
|
|||
If present, this will be prefixed to all of the keys before
|
||||
transmitting to the statsd server.
|
||||
|
||||
ZooKeeper
|
||||
~~~~~~~~~
|
||||
|
||||
.. attr:: zookeeper
|
||||
|
||||
Client connection information for ZooKeeper. TLS is required.
|
||||
|
|
@ -223,6 +132,45 @@ The following sections of ``zuul.conf`` are used by all Zuul components:
|
|||
|
||||
The ZooKeeper session timeout, in seconds.
|
||||
|
||||
|
||||
.. _database:
|
||||
|
||||
Database
|
||||
~~~~~~~~
|
||||
|
||||
.. attr:: database
|
||||
|
||||
.. attr:: dburi
|
||||
:required:
|
||||
|
||||
Database connection information in the form of a URI understood
|
||||
by SQLAlchemy. See `The SQLAlchemy manual
|
||||
<https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>`_
|
||||
for more information.
|
||||
|
||||
The driver will automatically set up the database creating and managing
|
||||
the necessary tables. Therefore the provided user should have sufficient
|
||||
permissions to manage the database. For example:
|
||||
|
||||
.. code-block:: sql
|
||||
|
||||
GRANT ALL ON my_database TO 'my_user'@'%';
|
||||
|
||||
.. attr:: pool_recycle
|
||||
:default: 1
|
||||
|
||||
Tune the pool_recycle value. See `The SQLAlchemy manual on pooling
|
||||
<http://docs.sqlalchemy.org/en/latest/core/pooling.html#setting-pool-recycle>`_
|
||||
for more information.
|
||||
|
||||
.. attr:: table_prefix
|
||||
:default: ''
|
||||
|
||||
The string to prefix the table names. This makes it possible to run
|
||||
several zuul deployments against the same database. This can be useful
|
||||
if you rely on external databases which are not under your control.
|
||||
The default is to have no prefix.
|
||||
|
||||
.. _scheduler:
|
||||
|
||||
Scheduler
|
||||
|
|
@ -252,9 +200,6 @@ by the Executors.
|
|||
It must also be able to connect to any services for which connections
|
||||
are configured (Gerrit, GitHub, etc).
|
||||
|
||||
Configuration
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
The following sections of ``zuul.conf`` are used by the scheduler:
|
||||
|
||||
|
||||
|
|
@ -412,31 +357,6 @@ The following sections of ``zuul.conf`` are used by the scheduler:
|
|||
To use IPv6, python>3.8 is required `issue24209 <https://bugs.python.org/issue24209>`_.
|
||||
|
||||
|
||||
Operation
|
||||
~~~~~~~~~
|
||||
|
||||
To start the scheduler, run ``zuul-scheduler``. To stop it, run
|
||||
``zuul-scheduler stop``.
|
||||
|
||||
Reconfiguration
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
Most of Zuul's configuration is automatically updated as changes to
|
||||
the repositories which contain it are merged. However, Zuul must be
|
||||
explicitly notified of changes to the tenant config file, since it is
|
||||
not read from a git repository. Zuul supports two kinds of reconfigurations.
|
||||
|
||||
The full reconfiguration refetches and reloads the configuration of
|
||||
all tenants. To do so, run ``zuul-scheduler full-reconfigure``. For
|
||||
example this can be used to fix eventual configuration inconsistencies
|
||||
after connection problems to Gerrit/Github.
|
||||
|
||||
The smart reconfiguration reloads only the tenants that changed their
|
||||
configuration in the tenant config file. To do so, run
|
||||
``zuul-scheduler smart-reconfigure``. In multi tenant systems this can
|
||||
be much faster than the full reconfiguration so it is recommended to
|
||||
use the smart reconfiguration after changing the tenant configuration
|
||||
file.
|
||||
|
||||
Merger
|
||||
------
|
||||
|
|
@ -461,9 +381,6 @@ Mergers need to be able to connect to the Gearman server (usually the
|
|||
scheduler host) as well as any services for which connections are
|
||||
configured (Gerrit, GitHub, etc).
|
||||
|
||||
Configuration
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
The following section of ``zuul.conf`` is used by the merger:
|
||||
|
||||
.. attr:: merger
|
||||
|
|
@ -523,17 +440,6 @@ The following section of ``zuul.conf`` is used by the merger:
|
|||
|
||||
Path to PID lock file for the merger process.
|
||||
|
||||
Operation
|
||||
~~~~~~~~~
|
||||
|
||||
To start the merger, run ``zuul-merger``.
|
||||
|
||||
In order to stop the merger and under normal circumstances it is
|
||||
best to pause and wait for all currently running tasks to finish
|
||||
before stopping it. To do so run ``zuul-merger pause``.
|
||||
|
||||
To stop the merger immediately, run ``zuul-merger stop``.
|
||||
|
||||
.. _executor:
|
||||
|
||||
Executor
|
||||
|
|
@ -935,70 +841,6 @@ The following sections of ``zuul.conf`` are used by the executor:
|
|||
to = user@example.org
|
||||
sender = zuul@example.org
|
||||
|
||||
Operation
|
||||
~~~~~~~~~
|
||||
|
||||
To start the executor, run ``zuul-executor``.
|
||||
|
||||
There are several commands which can be run to control the executor's
|
||||
behavior once it is running.
|
||||
|
||||
To pause the executor and prevent it from running new jobs you can
|
||||
run ``zuul-executor pause``.
|
||||
|
||||
To cause the executor to stop accepting new jobs and exit when all running
|
||||
jobs have finished you can run ``zuul-executor graceful``. Under most
|
||||
circumstances this will be the best way to stop Zuul.
|
||||
|
||||
To stop the executor immediately, run ``zuul-executor stop``. Jobs that were
|
||||
running on the stopped executor will be rescheduled on other executors.
|
||||
|
||||
The executor normally responds to a ``SIGTERM`` signal in the same way
|
||||
as the ``graceful`` command, however you can change this behavior to match
|
||||
``stop`` with the :attr:`executor.sigterm_method` setting.
|
||||
|
||||
To enable or disable running Ansible in verbose mode (with the
|
||||
``-vvv`` argument to ansible-playbook) run ``zuul-executor verbose``
|
||||
and ``zuul-executor unverbose``.
|
||||
|
||||
.. _ansible-and-python-3:
|
||||
|
||||
Ansible and Python 3
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
As noted above, the executor runs Ansible playbooks against the remote
|
||||
node(s) allocated for the job. Since part of executing playbooks on
|
||||
remote hosts is running Python scripts on them, Ansible needs to know
|
||||
what Python interpreter to use on the remote host. With older
|
||||
distributions, ``/usr/bin/python2`` was a generally sensible choice.
|
||||
However, over time a heterogeneous Python ecosystem has evolved where
|
||||
older distributions may only provide Python 2, most provide a mixed
|
||||
2/3 environment and newer distributions may only provide Python 3 (and
|
||||
then others like RHEL8 may even have separate "system" Python versions
|
||||
to add to confusion!).
|
||||
|
||||
Ansible's ``ansible_python_interpreter`` variable configures the path
|
||||
to the remote Python interpreter to use during playbook execution.
|
||||
This value is set by Zuul from the ``python-path`` specified for the
|
||||
node by Nodepool; see the `nodepool configuration documentation
|
||||
<https://zuul-ci.org/docs/nodepool/configuration.html>`__.
|
||||
|
||||
This defaults to ``auto``, where Ansible will automatically discover
|
||||
the interpreter available on the remote host. However, this setting
|
||||
only became available in Ansible >=2.8, so Zuul will translate
|
||||
``auto`` into the old default of ``/usr/bin/python2`` when configured
|
||||
to use older Ansible versions.
|
||||
|
||||
Thus for modern Python 3-only hosts no further configuration is needed
|
||||
when using Ansible >=2.8 (e.g. Fedora, Bionic onwards). If using
|
||||
earlier Ansible versions you may need to explicitly set the
|
||||
``python-path`` if ``/usr/bin/python2`` is not available on the node.
|
||||
|
||||
Ansible roles/modules which include Python code are generally Python 3
|
||||
safe now, but there is still a small possibility of incompatibility.
|
||||
See also the Ansible `Python 3 support page
|
||||
<https://docs.ansible.com/ansible/latest/reference_appendices/python_3_support.html>`__.
|
||||
|
||||
.. _web-server:
|
||||
|
||||
Web Server
|
||||
|
|
@ -1018,9 +860,6 @@ able to connect to the database it reports to in order to support the
|
|||
dashboard. If a GitHub connection is configured, they need to be
|
||||
reachable by GitHub so they may receive notifications.
|
||||
|
||||
Configuration
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
In addition to the common configuration sections, the following
|
||||
sections of ``zuul.conf`` are used by the web server:
|
||||
|
||||
|
|
@ -1087,10 +926,9 @@ sections of ``zuul.conf`` are used by the web server:
|
|||
|
||||
If this is used the finger gateways should be configured accordingly.
|
||||
|
||||
.. _web-server-tenant-scoped-api:
|
||||
|
||||
Enabling tenant-scoped access to privileged actions
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
Authentication
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
A user can be granted access to protected REST API endpoints by providing a
|
||||
valid JWT (JSON Web Token) as a bearer token when querying the API endpoints.
|
||||
|
|
@ -1235,22 +1073,13 @@ authentication on Zuul's web user interface.
|
|||
under the key "jwks_uri", therefore this attribute is usually not necessary.
|
||||
|
||||
|
||||
Operation
|
||||
~~~~~~~~~
|
||||
|
||||
To start the web server, run ``zuul-web``. To stop it, kill the
|
||||
PID which was saved in the pidfile specified in the configuration.
|
||||
|
||||
Web Client
|
||||
----------
|
||||
Client
|
||||
------
|
||||
|
||||
Zuul's command line client may be configured to make calls to Zuul's web
|
||||
server. The client will then look for a ``zuul.conf`` file with a ``webclient``
|
||||
section to set up the connection over HTTP.
|
||||
|
||||
Configuration
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
.. attr:: webclient
|
||||
|
||||
.. attr:: url
|
||||
|
|
@ -1263,22 +1092,6 @@ Configuration
|
|||
Enforce SSL verification when sending requests over to Zuul's web server.
|
||||
This should only be disabled when working with test servers.
|
||||
|
||||
Configuration
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
In addition to the common configuration sections, the following
|
||||
sections of ``zuul.conf`` are used by the web server:
|
||||
|
||||
.. attr:: web
|
||||
|
||||
.. attr:: listen_address
|
||||
:default: 127.0.0.1
|
||||
|
||||
IP address or domain name on which to listen.
|
||||
|
||||
.. attr:: log_config
|
||||
|
||||
Path to log config file for the web server process.
|
||||
|
||||
Finger Gateway
|
||||
--------------
|
||||
|
|
@ -1311,9 +1124,6 @@ the following purposes:
|
|||
zuul-web will route to the executors via finger gateways in the same
|
||||
zone.
|
||||
|
||||
Configuration
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
In addition to the common configuration sections, the following
|
||||
sections of ``zuul.conf`` are used by the finger gateway:
|
||||
|
||||
|
|
@ -1417,8 +1227,39 @@ sections of ``zuul.conf`` are used by the finger gateway:
|
|||
but will still use the supplied certificate to make remote TLS
|
||||
connections.
|
||||
|
||||
Operation
|
||||
~~~~~~~~~
|
||||
.. _connections:
|
||||
|
||||
To start the finger gateway, run ``zuul-fingergw``. To stop it, kill the
|
||||
PID which was saved in the pidfile specified in the configuration.
|
||||
Connections
|
||||
===========
|
||||
|
||||
Most of Zuul's configuration is contained in the git repositories upon
|
||||
which Zuul operates, however, some configuration outside of git
|
||||
repositories is still required to bootstrap the system. This includes
|
||||
information on connections between Zuul and other systems, as well as
|
||||
identifying the projects Zuul uses.
|
||||
|
||||
In order to interact with external systems, Zuul must have a
|
||||
*connection* to that system configured. Zuul includes a number of
|
||||
:ref:`drivers <drivers>`, each of which implements the functionality
|
||||
necessary to connect to a system. Each connection in Zuul is
|
||||
associated with a driver.
|
||||
|
||||
To configure a connection in Zuul, select a unique name for the
|
||||
connection and add a section to ``zuul.conf`` with the form
|
||||
``[connection NAME]``. For example, a connection to a gerrit server
|
||||
may appear as:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[connection mygerritserver]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
|
||||
Zuul needs to use a single connection to look up information about
|
||||
changes hosted by a given system. When it looks up changes, it will
|
||||
do so using the first connection it finds that matches the server name
|
||||
it's looking for. It's generally best to use only a single connection
|
||||
for a given server, however, if you need more than one (for example,
|
||||
to satisfy unique reporting requirements) be sure to list the primary
|
||||
connection first as that is what Zuul will use to look up all changes
|
||||
for that server.
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
:title: Encryption
|
||||
|
||||
.. _encryption:
|
||||
|
||||
Encryption
|
||||
==========
|
||||
|
||||
Zuul supports storing encrypted data directly in the git repositories
|
||||
of projects it operates on. If you have a job which requires private
|
||||
information in order to run (e.g., credentials to interact with a
|
||||
third-party service) those credentials can be stored along with the
|
||||
job definition.
|
||||
|
||||
Each project in Zuul has its own automatically generated RSA keypair
|
||||
which can be used by anyone to encrypt a secret and only Zuul is able
|
||||
to decrypt it. Zuul serves each project's public key using its
|
||||
build-in webserver. They can be fetched at the path
|
||||
``/api/tenant/<tenant>/key/<project>.pub`` where ``<project>`` is the
|
||||
canonical name of a project and ``<tenant>`` is the name of a tenant
|
||||
with that project.
|
||||
|
||||
Zuul currently supports one encryption scheme, PKCS#1 with OAEP, which
|
||||
can not store secrets longer than the 3760 bits (derived from the key
|
||||
length of 4096 bits minus 336 bits of overhead). The padding used by
|
||||
this scheme ensures that someone examining the encrypted data can not
|
||||
determine the length of the plaintext version of the data, except to
|
||||
know that it is not longer than 3760 bits (or some multiple thereof).
|
||||
|
||||
In the config files themselves, Zuul uses an extensible method of
|
||||
specifying the encryption scheme used for a secret so that other
|
||||
schemes may be added later. To specify a secret, use the
|
||||
``!encrypted/pkcs1-oaep`` YAML tag along with the base64 encoded
|
||||
value. For example:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
- secret:
|
||||
name: test_secret
|
||||
data:
|
||||
password: !encrypted/pkcs1-oaep |
|
||||
BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi
|
||||
...
|
||||
|
||||
To support secrets longer than 3760 bits, the value after the
|
||||
encryption tag may be a list rather than a scalar. For example:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
- secret:
|
||||
name: long_secret
|
||||
data:
|
||||
password: !encrypted/pkcs1-oaep
|
||||
- er1UXNOD3OqtsRJaP0Wvaqiqx0ZY2zzRt6V9vqIsRaz1R5C4/AEtIad/DERZHwk3Nk+KV
|
||||
...
|
||||
- HdWDS9lCBaBJnhMsm/O9tpzCq+GKRELpRzUwVgU5k822uBwhZemeSrUOLQ8hQ7q/vVHln
|
||||
...
|
||||
|
||||
The `zuul-client utility <https://zuul-ci.org/docs/zuul-client/>`_ provides a
|
||||
simple way to encrypt secrets for a Zuul project:
|
||||
|
||||
.. program-output:: zuul-client encrypt --help
|
||||
|
|
@ -1,189 +0,0 @@
|
|||
:title: Github Checks API
|
||||
|
||||
Github Checks API
|
||||
=================
|
||||
|
||||
Github provides two distinct methods for reporting results; a "checks"
|
||||
and a "status" API.
|
||||
|
||||
The `checks API`_ provides some additional features compared to the
|
||||
`status API`_ like file comments and custom actions (e.g. cancel a
|
||||
running build).
|
||||
|
||||
Either can be chosen when configuring Zuul to report for your Github
|
||||
project. However, there are some considerations to take into account
|
||||
when choosing the API.
|
||||
|
||||
Design decisions
|
||||
----------------
|
||||
|
||||
The Github checks API defines the concepts of `Check Suites`_ and
|
||||
`Check Runs`_. *Check suites* are a collection of *check runs* for a
|
||||
specific commit and summarize a final status
|
||||
|
||||
A priori the check suite appears to be a good mapping for a pipeline
|
||||
execution in Zuul, where a check run maps to a single job execution
|
||||
that is part of the pipeline run. Unfortunately, there are a few
|
||||
problematic restrictions mapping between Github and Zuul concepts.
|
||||
|
||||
Github check suites are opaque and the current status, duration and
|
||||
the overall conclusion are all calculated and set automatically
|
||||
whenever an included check run is updated. Most importantly, there
|
||||
can only be one check suite per commit SHA, per app. Thus there is no
|
||||
facility for for Zuul to create multiple check suite results for a
|
||||
change, e.g. one check suite for each pipeline such as check and gate.
|
||||
|
||||
The Github check suite thus does not map well to Zuul's concept of
|
||||
multiple pipelines for a single change. Since a check suite is unique
|
||||
and global for the change, it can not be used to flag the status of
|
||||
arbitrary pipelines. This makes the check suite API insufficient for
|
||||
recording details that Zuul needs such as "the check pipeline has
|
||||
passed but the gate pipeline has failed".
|
||||
|
||||
Another issue is that Zuul only reports on the results of the whole
|
||||
pipeline, not individual jobs. Reporting each Zuul job as a separate
|
||||
check is problematic for a number of reasons.
|
||||
|
||||
Zuul often runs the same job for the same change multiple times; for
|
||||
example in the check and gate pipeline. There is no facility for
|
||||
these runs to be reported differently in the single check suite for
|
||||
the Github change.
|
||||
|
||||
When configuring branch protection in Github, only a *check run* can
|
||||
be selected as required status check. This is in conflict with
|
||||
managing jobs in pipelines with Zuul. For example, to implement
|
||||
branch protection on GitHub would mean listing each job as a dedicated
|
||||
check, leading to a check run list that is not kept in sync with the
|
||||
project's Zuul pipeline configuration. Additionally, you lose some
|
||||
of Zuul's features like non-voting jobs as Github branch protections
|
||||
has no concept of a non-voting job.
|
||||
|
||||
Thus Zuul can integrate with the checks API, but only at a pipeline
|
||||
level. Each pipeline execution will map to a check-run result
|
||||
reported to Github.
|
||||
|
||||
Behaviour in Zuul
|
||||
-----------------
|
||||
|
||||
Reporting
|
||||
~~~~~~~~~
|
||||
|
||||
The Github reporter is able to report both a status
|
||||
:attr:`pipeline.<reporter>.<github source>.status` or a check
|
||||
:attr:`pipeline.<reporter>.<github source>.check`. While it's possible to
|
||||
configure a Github reporter to report both, it's recommended to use only one.
|
||||
Reporting both might result in duplicated status check entries in the Github
|
||||
PR (the section below the comments).
|
||||
|
||||
Trigger
|
||||
~~~~~~~
|
||||
|
||||
The Github driver is able to trigger on a reported check
|
||||
(:value:`pipeline.trigger.<github source>.event.check_run`) similar to a
|
||||
reported status (:value:`pipeline.trigger.<github source>.action.status`).
|
||||
|
||||
Requirements
|
||||
~~~~~~~~~~~~
|
||||
|
||||
While trigger and reporter differentiates between status and check, the Github
|
||||
driver does not differentiate between them when it comes to pipeline
|
||||
requirements. This is mainly because Github also doesn't differentiate between
|
||||
both in terms of branch protection and `status checks`_.
|
||||
|
||||
Actions / Events
|
||||
----------------
|
||||
|
||||
Github provides a set of default actions for check suites and check runs.
|
||||
Those actions are available as buttons in the Github UI. Clicking on those
|
||||
buttons will emit webhook events which will be handled by Zuul.
|
||||
|
||||
These actions are only available on failed check runs / check suites. So
|
||||
far, a running or successful check suite / check run does not provide any
|
||||
action from Github side.
|
||||
|
||||
Available actions are:
|
||||
|
||||
Re-run all checks
|
||||
Github emits a webhook event with type ``check_suite`` and action
|
||||
``rerequested`` that is meant to re-run all check-runs contained in this
|
||||
check suite. Github does not provide the list of check-runs in that case,
|
||||
so it's up to the Github app what should run.
|
||||
|
||||
Re-run failed checks
|
||||
Github emits a webhook event with type ``check_run`` and action
|
||||
``rerequested`` for each failed check run contained in this suite.
|
||||
|
||||
Re-run
|
||||
Github emits a webhook event with type ``check_run`` and action
|
||||
``rerequested`` for the specific check run.
|
||||
|
||||
Zuul will handle all events except for the `Re-run all checks` event;
|
||||
it does not make sense in the Zuul model to trigger all pipelines to
|
||||
run simultaneously.
|
||||
|
||||
These events are unable to be customized in Github. Github will
|
||||
always report "You have successfully requested ..." despite nothing
|
||||
listening to the event. Therefore, it might be a solution to handle
|
||||
the `Re-run all checks` event in Zuul similar to `Re-run failed
|
||||
checks` just to not do anything while Github makes the user believe an
|
||||
action was really triggered.
|
||||
|
||||
|
||||
File comments (annotations)
|
||||
---------------------------
|
||||
|
||||
Check runs can be used to post file comments directly in the files of the PR.
|
||||
Those are similar to user comments, but must provide some more information.
|
||||
|
||||
Zuul jobs can already return file comments via ``zuul_return``
|
||||
(see: :ref:`return_values`). We can simply use this return value, build the
|
||||
necessary annotations (how Github calls it) from it and attach them to the
|
||||
check run.
|
||||
|
||||
|
||||
Custom actions
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
Check runs can provide some custom actions which will result in additional
|
||||
buttons being available in the Github UI for this specific check run.
|
||||
Clicking on such a button will emit a webhook event with type ``check_run``
|
||||
and action ``requested_action`` and will additionally contain the id/name of
|
||||
the requested action which we can define when creating the action on the
|
||||
check run.
|
||||
|
||||
We could use these custom actions to provide some "Re-run" action on a
|
||||
running check run (which might otherwise be stuck in case a check run update
|
||||
fails) or to abort a check run directly from the Github UI.
|
||||
|
||||
|
||||
Restrictions and Recommendations
|
||||
--------------------------------
|
||||
|
||||
Although both the checks API and the status API can be activated for a
|
||||
Github reporter at the same time, it's not recommended to do so as this might
|
||||
result in multiple status checks to be reported to the PR for the same pipeline
|
||||
execution (which would result in duplicated entries in the status section below
|
||||
the comments of a PR).
|
||||
|
||||
In case the update on a check run fails (e.g. request timeout when reporting
|
||||
success or failure to Github), the check run will stay in status "in_progess"
|
||||
and there will be no way to re-run the check run via the Github UI as the
|
||||
predefined actions are only available on failed check runs.
|
||||
Thus, it's recommended to configure a
|
||||
:value:`pipeline.trigger.<github source>.action.comment` trigger on the
|
||||
pipeline to still be able to trigger re-run of the stuck check run via e.g.
|
||||
"recheck".
|
||||
|
||||
The check suite will only list check runs that were reported by Zuul. If
|
||||
the requirements for a certain pipeline are not met and it is not run, the
|
||||
check run for this pipeline won't be listed in the check suite. However,
|
||||
this does not affect the required status checks. If the check run is enabled
|
||||
as required, Github will still show it in the list of required status checks
|
||||
- even if it didn't run yet - just not in the check suite.
|
||||
|
||||
|
||||
.. _checks API: https://docs.github.com/v3/checks/
|
||||
.. _status API: https://docs.github.com/v3/repos/statuses/
|
||||
.. _Check Suites: https://docs.github.com/v3/checks/suites/
|
||||
.. _Check Runs: https://docs.github.com/v3/checks/runs/
|
||||
.. _status checks: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-status-checks#types-of-status-checks-on-github
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
Discussion
|
||||
==========
|
||||
|
||||
This guide is for all users of Zuul. If you work on a project where
|
||||
Zuul is used to drive automation (whether that's testing proposed
|
||||
changes, building artifacts, or deploying builds), this guide will
|
||||
help you understand the concepts that underlie Zuul, and how to
|
||||
configure it to meet your needs.
|
||||
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
concepts
|
||||
components
|
||||
gating
|
||||
encryption
|
||||
tenant-scoped-rest-api
|
||||
github-checks-api
|
||||
|
|
@ -600,3 +600,191 @@ Here is an example of standard pipelines you may want to define:
|
|||
|
||||
.. literalinclude:: /examples/pipelines/github-reference-pipelines.yaml
|
||||
:language: yaml
|
||||
|
||||
Github Checks API
|
||||
-----------------
|
||||
|
||||
Github provides two distinct methods for reporting results; a "checks"
|
||||
and a "status" API.
|
||||
|
||||
The `checks API`_ provides some additional features compared to the
|
||||
`status API`_ like file comments and custom actions (e.g. cancel a
|
||||
running build).
|
||||
|
||||
Either can be chosen when configuring Zuul to report for your Github
|
||||
project. However, there are some considerations to take into account
|
||||
when choosing the API.
|
||||
|
||||
Design decisions
|
||||
................
|
||||
|
||||
The Github checks API defines the concepts of `Check Suites`_ and
|
||||
`Check Runs`_. *Check suites* are a collection of *check runs* for a
|
||||
specific commit and summarize a final status
|
||||
|
||||
A priori the check suite appears to be a good mapping for a pipeline
|
||||
execution in Zuul, where a check run maps to a single job execution
|
||||
that is part of the pipeline run. Unfortunately, there are a few
|
||||
problematic restrictions mapping between Github and Zuul concepts.
|
||||
|
||||
Github check suites are opaque and the current status, duration and
|
||||
the overall conclusion are all calculated and set automatically
|
||||
whenever an included check run is updated. Most importantly, there
|
||||
can only be one check suite per commit SHA, per app. Thus there is no
|
||||
facility for for Zuul to create multiple check suite results for a
|
||||
change, e.g. one check suite for each pipeline such as check and gate.
|
||||
|
||||
The Github check suite thus does not map well to Zuul's concept of
|
||||
multiple pipelines for a single change. Since a check suite is unique
|
||||
and global for the change, it can not be used to flag the status of
|
||||
arbitrary pipelines. This makes the check suite API insufficient for
|
||||
recording details that Zuul needs such as "the check pipeline has
|
||||
passed but the gate pipeline has failed".
|
||||
|
||||
Another issue is that Zuul only reports on the results of the whole
|
||||
pipeline, not individual jobs. Reporting each Zuul job as a separate
|
||||
check is problematic for a number of reasons.
|
||||
|
||||
Zuul often runs the same job for the same change multiple times; for
|
||||
example in the check and gate pipeline. There is no facility for
|
||||
these runs to be reported differently in the single check suite for
|
||||
the Github change.
|
||||
|
||||
When configuring branch protection in Github, only a *check run* can
|
||||
be selected as required status check. This is in conflict with
|
||||
managing jobs in pipelines with Zuul. For example, to implement
|
||||
branch protection on GitHub would mean listing each job as a dedicated
|
||||
check, leading to a check run list that is not kept in sync with the
|
||||
project's Zuul pipeline configuration. Additionally, you lose some
|
||||
of Zuul's features like non-voting jobs as Github branch protections
|
||||
has no concept of a non-voting job.
|
||||
|
||||
Thus Zuul can integrate with the checks API, but only at a pipeline
|
||||
level. Each pipeline execution will map to a check-run result
|
||||
reported to Github.
|
||||
|
||||
Behaviour in Zuul
|
||||
.................
|
||||
|
||||
Reporting
|
||||
~~~~~~~~~
|
||||
|
||||
The Github reporter is able to report both a status
|
||||
:attr:`pipeline.<reporter>.<github source>.status` or a check
|
||||
:attr:`pipeline.<reporter>.<github source>.check`. While it's possible to
|
||||
configure a Github reporter to report both, it's recommended to use only one.
|
||||
Reporting both might result in duplicated status check entries in the Github
|
||||
PR (the section below the comments).
|
||||
|
||||
Trigger
|
||||
~~~~~~~
|
||||
|
||||
The Github driver is able to trigger on a reported check
|
||||
(:value:`pipeline.trigger.<github source>.event.check_run`) similar to a
|
||||
reported status (:value:`pipeline.trigger.<github source>.action.status`).
|
||||
|
||||
Requirements
|
||||
~~~~~~~~~~~~
|
||||
|
||||
While trigger and reporter differentiates between status and check, the Github
|
||||
driver does not differentiate between them when it comes to pipeline
|
||||
requirements. This is mainly because Github also doesn't differentiate between
|
||||
both in terms of branch protection and `status checks`_.
|
||||
|
||||
Actions / Events
|
||||
................
|
||||
|
||||
Github provides a set of default actions for check suites and check runs.
|
||||
Those actions are available as buttons in the Github UI. Clicking on those
|
||||
buttons will emit webhook events which will be handled by Zuul.
|
||||
|
||||
These actions are only available on failed check runs / check suites. So
|
||||
far, a running or successful check suite / check run does not provide any
|
||||
action from Github side.
|
||||
|
||||
Available actions are:
|
||||
|
||||
Re-run all checks
|
||||
Github emits a webhook event with type ``check_suite`` and action
|
||||
``rerequested`` that is meant to re-run all check-runs contained in this
|
||||
check suite. Github does not provide the list of check-runs in that case,
|
||||
so it's up to the Github app what should run.
|
||||
|
||||
Re-run failed checks
|
||||
Github emits a webhook event with type ``check_run`` and action
|
||||
``rerequested`` for each failed check run contained in this suite.
|
||||
|
||||
Re-run
|
||||
Github emits a webhook event with type ``check_run`` and action
|
||||
``rerequested`` for the specific check run.
|
||||
|
||||
Zuul will handle all events except for the `Re-run all checks` event;
|
||||
it does not make sense in the Zuul model to trigger all pipelines to
|
||||
run simultaneously.
|
||||
|
||||
These events are unable to be customized in Github. Github will
|
||||
always report "You have successfully requested ..." despite nothing
|
||||
listening to the event. Therefore, it might be a solution to handle
|
||||
the `Re-run all checks` event in Zuul similar to `Re-run failed
|
||||
checks` just to not do anything while Github makes the user believe an
|
||||
action was really triggered.
|
||||
|
||||
|
||||
File comments (annotations)
|
||||
...........................
|
||||
|
||||
Check runs can be used to post file comments directly in the files of the PR.
|
||||
Those are similar to user comments, but must provide some more information.
|
||||
|
||||
Zuul jobs can already return file comments via ``zuul_return``
|
||||
(see: :ref:`return_values`). We can simply use this return value, build the
|
||||
necessary annotations (how Github calls it) from it and attach them to the
|
||||
check run.
|
||||
|
||||
|
||||
Custom actions
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
Check runs can provide some custom actions which will result in additional
|
||||
buttons being available in the Github UI for this specific check run.
|
||||
Clicking on such a button will emit a webhook event with type ``check_run``
|
||||
and action ``requested_action`` and will additionally contain the id/name of
|
||||
the requested action which we can define when creating the action on the
|
||||
check run.
|
||||
|
||||
We could use these custom actions to provide some "Re-run" action on a
|
||||
running check run (which might otherwise be stuck in case a check run update
|
||||
fails) or to abort a check run directly from the Github UI.
|
||||
|
||||
|
||||
Restrictions and Recommendations
|
||||
................................
|
||||
|
||||
Although both the checks API and the status API can be activated for a
|
||||
Github reporter at the same time, it's not recommended to do so as this might
|
||||
result in multiple status checks to be reported to the PR for the same pipeline
|
||||
execution (which would result in duplicated entries in the status section below
|
||||
the comments of a PR).
|
||||
|
||||
In case the update on a check run fails (e.g. request timeout when reporting
|
||||
success or failure to Github), the check run will stay in status "in_progess"
|
||||
and there will be no way to re-run the check run via the Github UI as the
|
||||
predefined actions are only available on failed check runs.
|
||||
Thus, it's recommended to configure a
|
||||
:value:`pipeline.trigger.<github source>.action.comment` trigger on the
|
||||
pipeline to still be able to trigger re-run of the stuck check run via e.g.
|
||||
"recheck".
|
||||
|
||||
The check suite will only list check runs that were reported by Zuul. If
|
||||
the requirements for a certain pipeline are not met and it is not run, the
|
||||
check run for this pipeline won't be listed in the check suite. However,
|
||||
this does not affect the required status checks. If the check run is enabled
|
||||
as required, Github will still show it in the list of required status checks
|
||||
- even if it didn't run yet - just not in the check suite.
|
||||
|
||||
|
||||
.. _checks API: https://docs.github.com/v3/checks/
|
||||
.. _status API: https://docs.github.com/v3/repos/statuses/
|
||||
.. _Check Suites: https://docs.github.com/v3/checks/suites/
|
||||
.. _Check Runs: https://docs.github.com/v3/checks/runs/
|
||||
.. _status checks: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-status-checks#types-of-status-checks-on-github
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
Admin How-to Guides
|
||||
===================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
installation
|
||||
zuul-from-scratch
|
||||
openid-connect-examples
|
||||
troubleshooting
|
||||
zookeeper
|
||||
|
|
@ -1,10 +1,12 @@
|
|||
User How-to Guides
|
||||
==================
|
||||
How-To Guides
|
||||
=============
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
zuul-from-scratch
|
||||
cross-project-gating
|
||||
pti
|
||||
badges
|
||||
matrix
|
||||
zookeeper
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
OpenID Connect Integration Examples
|
||||
===================================
|
||||
|
||||
This document lists simple How-Tos to help administrators enable OpenID
|
||||
Connect authentication in Zuul and Zuul's Web UI.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
openid-with-google
|
||||
openid-with-keycloak
|
||||
|
||||
Debugging
|
||||
---------
|
||||
|
||||
If problems appear:
|
||||
|
||||
* Make sure your configuration is correct, especially callback URIs.
|
||||
* More information can be found in Zuul's web service logs.
|
||||
* From the user's side, activating the web console in the browser can be helpful
|
||||
to debug API calls.
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
.. _howto-zookeeper:
|
||||
|
||||
ZooKeeper Administration
|
||||
========================
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ install Zuul without the aid of any existing installation tools, you
|
|||
may find this a useful reference.
|
||||
|
||||
If, instead, you want to get Zuul running quickly, see the
|
||||
:ref:`quick_start` which runs all of the Zuul services in containers
|
||||
:ref:`quick-start` which runs all of the Zuul services in containers
|
||||
with a single command.
|
||||
|
||||
Environment Setup
|
||||
|
|
|
|||
|
|
@ -13,41 +13,35 @@ If you are looking for the Javascript testing tool named Zuul, it
|
|||
can be found here:
|
||||
https://github.com/defunctzombie/zuul
|
||||
|
||||
Documentation
|
||||
-------------
|
||||
|
||||
Zuul Users
|
||||
**********
|
||||
How To Use This Manual
|
||||
----------------------
|
||||
|
||||
If you have access to a Zuul system operated by someone else, then you
|
||||
may be interested in :ref:`about-zuul` and the following reference
|
||||
sections: :ref:`project-configuration` and :ref:`job-content`.
|
||||
|
||||
If you would like to learn how to run Zuul, try the :ref:`quick-start`.
|
||||
|
||||
If you are or will be responsible for installing and operating a Zuul
|
||||
System, the remainder of the sections will be useful.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:includehidden:
|
||||
:maxdepth: 1
|
||||
|
||||
tutorials/user
|
||||
howtos/user
|
||||
reference/user
|
||||
|
||||
Zuul Admins
|
||||
***********
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
tutorials/admin
|
||||
howtos/admin
|
||||
reference/admin
|
||||
|
||||
Digging Deeper
|
||||
**************
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
discussion/index
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
reference/index
|
||||
about
|
||||
tutorials/quick-start
|
||||
project-config
|
||||
job-content
|
||||
admin
|
||||
rest-api
|
||||
howtos/index
|
||||
developer/index
|
||||
governance
|
||||
vulnerabilities
|
||||
releasenotes
|
||||
glossary
|
||||
|
||||
Indices and tables
|
||||
------------------
|
||||
|
|
|
|||
|
|
@ -1,61 +1,5 @@
|
|||
Installation Reference
|
||||
======================
|
||||
|
||||
Install Zuul
|
||||
------------
|
||||
|
||||
To install a Zuul release from PyPI, run::
|
||||
|
||||
pip install zuul
|
||||
|
||||
Or from a git checkout, run::
|
||||
|
||||
pip install .
|
||||
|
||||
That will also install Zuul's python dependencies. To minimize
|
||||
interaction with other python packages installed on a system, you may
|
||||
wish to install Zuul within a Python virtualenv.
|
||||
|
||||
Zuul has several system-level dependencies as well. You can find a
|
||||
list of operating system packages in ``bindep.txt`` in Zuul's source
|
||||
directory.
|
||||
|
||||
It is further required to run ``zuul-manage-ansible`` on the zuul-executor
|
||||
in order to install all supported ansible versions so zuul can use them.
|
||||
See :ref:`ansible-installation-options` for details.
|
||||
|
||||
Zuul Components
|
||||
---------------
|
||||
|
||||
Zuul provides the following components:
|
||||
|
||||
- **zuul-scheduler**: The main Zuul process. Handles receiving
|
||||
events, executing jobs, collecting results and posting reports.
|
||||
Coordinates the work of the other components. It also provides
|
||||
a gearman daemon which the other components use for
|
||||
coordination.
|
||||
|
||||
- **zuul-merger**: Scale-out component that performs git merge
|
||||
operations. Zuul performs a large number of git operations in
|
||||
the course of its work. Adding merger processes can help speed
|
||||
Zuul's processing. This component is optional (zero or more of
|
||||
these can be run).
|
||||
|
||||
- **zuul-executor**: Scale-out component for executing jobs. At
|
||||
least one of these is required. Depending on system
|
||||
configuration, you can expect a single executor to handle up to
|
||||
about 100 simultaneous jobs. Can handle the functions of a
|
||||
merger if dedicated mergers are not provided. One or more of
|
||||
these must be run.
|
||||
|
||||
- **zuul-web**: A web server that receives "webhook" events from
|
||||
external providers, supplies a web dashboard, and provides
|
||||
websocket access to live streaming of logs.
|
||||
|
||||
- **zuul-fingergw**: A gateway which provides finger protocol
|
||||
access to live streaming of logs.
|
||||
|
||||
For more detailed information about these, see :ref:`components`.
|
||||
Installation
|
||||
============
|
||||
|
||||
External Dependencies
|
||||
---------------------
|
||||
|
|
@ -91,19 +35,69 @@ in given a username and ssh private key.
|
|||
ZooKeeper
|
||||
~~~~~~~~~
|
||||
|
||||
.. TODO: SpamapS any zookeeper config recommendations?
|
||||
|
||||
Nodepool uses ZooKeeper to communicate internally among its
|
||||
components, and also to communicate with Zuul. You can run a simple
|
||||
single-node ZooKeeper instance, or a multi-node cluster. Ensure that
|
||||
the host running the Zuul scheduler has access to the cluster.
|
||||
the host running the Zuul scheduler has access to the cluster. See
|
||||
:ref:`howto-zookeeper` for recommendations for operating a small
|
||||
ZooKeeper cluster.
|
||||
|
||||
Zuul stores private keys for each project it knows about in ZooKeeper.
|
||||
It is recommended that you periodically back up the private keys in
|
||||
case the ZooKeeper data store is lost or damaged. The :title:`Zuul
|
||||
Client` provides two sub-commands for use in this case:
|
||||
:title:`export-keys` and :title:`import-keys`. Each takes an argument to
|
||||
a filesystem path and will write the keys to, or read the keys from
|
||||
that path. The data in the exported files are still secured with the
|
||||
keystore passphrase, so be sure to retain it as well.
|
||||
|
||||
Database
|
||||
~~~~~~~~
|
||||
|
||||
Zuul requires an SQL database; either MariaDB, MySQL, or PostgreSQL.
|
||||
|
||||
Installation from PyPI
|
||||
----------------------
|
||||
|
||||
Zuul is a Python application which can be installed from the Python
|
||||
Package Index (PyPI).
|
||||
|
||||
To install a Zuul release from PyPI, run::
|
||||
|
||||
pip install zuul
|
||||
|
||||
Or from a git checkout, run::
|
||||
|
||||
pip install .
|
||||
|
||||
That will also install Zuul's python dependencies. To minimize
|
||||
interaction with other python packages installed on a system, you may
|
||||
wish to install Zuul within a Python virtualenv.
|
||||
|
||||
Zuul has several system-level dependencies as well. You can find a
|
||||
list of operating system packages in ``bindep.txt`` in Zuul's source
|
||||
directory.
|
||||
|
||||
It is also required to run ``zuul-manage-ansible`` on the
|
||||
zuul-executor in order to install all supported Ansible versions so
|
||||
Zuul can use them. See :ref:`ansible-installation-options` for
|
||||
details.
|
||||
|
||||
Installation from Containers
|
||||
----------------------------
|
||||
|
||||
The Zuul project also builds and releases container images on
|
||||
DockerHub. These are available at: https://hub.docker.com/u/zuul
|
||||
|
||||
There is a container image for each of the Zuul :ref:`components <components>`.
|
||||
|
||||
.. _ansible-installation-options:
|
||||
|
||||
Ansible
|
||||
~~~~~~~
|
||||
Executor Deployment
|
||||
-------------------
|
||||
|
||||
There are two approaches that can be used to install Ansible for Zuul.
|
||||
The Zuul executor requires Ansible to run jobs. There are two
|
||||
approaches that can be used to install Ansible for Zuul.
|
||||
|
||||
First you may set ``manage_ansible`` to True in the executor config. If you
|
||||
do this Zuul will install all supported Ansible versions on zuul-executor
|
||||
|
|
@ -122,79 +116,10 @@ state dirs).
|
|||
In both cases if using a non default path you will want to set
|
||||
``ansible_root`` in the executor config file.
|
||||
|
||||
Zuul Setup
|
||||
----------
|
||||
|
||||
At minimum you need to provide ``zuul.conf`` and ``main.yaml`` placed
|
||||
in ``/etc/zuul/``. The following example uses the builtin gearman
|
||||
service in Zuul, and a connection to Gerrit.
|
||||
|
||||
**zuul.conf**::
|
||||
|
||||
[keystore]
|
||||
password=secret
|
||||
|
||||
[scheduler]
|
||||
tenant_config=/etc/zuul/main.yaml
|
||||
|
||||
[gearman_server]
|
||||
start=true
|
||||
|
||||
[gearman]
|
||||
server=127.0.0.1
|
||||
|
||||
[connection my_gerrit]
|
||||
driver=gerrit
|
||||
server=git.example.com
|
||||
port=29418
|
||||
baseurl=https://git.example.com/gerrit/
|
||||
user=zuul
|
||||
sshkey=/home/zuul/.ssh/id_rsa
|
||||
|
||||
[database]
|
||||
dburi=mysql+pymysql://zuul:secret@mysql/zuul
|
||||
|
||||
See :ref:`components` and :ref:`connections` for more details.
|
||||
|
||||
The following tells Zuul to read its configuration from and operate on
|
||||
the *example-project* project:
|
||||
|
||||
**main.yaml**::
|
||||
|
||||
- tenant:
|
||||
name: example-tenant
|
||||
source:
|
||||
my_gerrit:
|
||||
untrusted-projects:
|
||||
- example-project
|
||||
|
||||
Starting Zuul
|
||||
-------------
|
||||
|
||||
You can run any zuul process with the **-d** option to make it not
|
||||
daemonize. It's a good idea at first to confirm there's no issues with
|
||||
your configuration.
|
||||
|
||||
To start, simply run::
|
||||
|
||||
zuul-scheduler
|
||||
|
||||
Once run you should have two zuul-scheduler processes (if using the
|
||||
built-in gearman server, or one process otherwise).
|
||||
|
||||
Before Zuul can run any jobs, it needs to load its configuration, most
|
||||
of which is in the git repositories that Zuul operates on. Start an
|
||||
executor to allow zuul to do that::
|
||||
|
||||
zuul-executor
|
||||
|
||||
Zuul should now be able to read its configuration from the configured
|
||||
repo and process any jobs defined therein.
|
||||
|
||||
.. _web-deployment-options:
|
||||
|
||||
Web Deployment Options
|
||||
----------------------
|
||||
Web Deployment
|
||||
--------------
|
||||
|
||||
The ``zuul-web`` service provides a web dashboard, a REST API and a websocket
|
||||
log streaming service as a single holistic web application. For production use
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
:title: Job Content
|
||||
|
||||
.. _job-content:
|
||||
|
||||
Job Content
|
||||
===========
|
||||
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
:title: Operation
|
||||
|
||||
Operation
|
||||
=========
|
||||
|
||||
You can run any zuul process with the **-d** option to make it not
|
||||
daemonize. It's a good idea at first to confirm there's no issues with
|
||||
your configuration.
|
||||
|
||||
To start, simply run::
|
||||
|
||||
zuul-scheduler
|
||||
|
||||
Once run you should have two zuul-scheduler processes (if using the
|
||||
built-in gearman server, or one process otherwise).
|
||||
|
||||
Before Zuul can run any jobs, it needs to load its configuration, most
|
||||
of which is in the git repositories that Zuul operates on. Start an
|
||||
executor to allow zuul to do that::
|
||||
|
||||
zuul-executor
|
||||
|
||||
Zuul should now be able to read its configuration from the configured
|
||||
repo and process any jobs defined therein.
|
||||
|
||||
Scheduler
|
||||
---------
|
||||
|
||||
Operation
|
||||
~~~~~~~~~
|
||||
|
||||
To start the scheduler, run ``zuul-scheduler``. To stop it, run
|
||||
``zuul-scheduler stop``.
|
||||
|
||||
Reconfiguration
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
Most of Zuul's configuration is automatically updated as changes to
|
||||
the repositories which contain it are merged. However, Zuul must be
|
||||
explicitly notified of changes to the tenant config file, since it is
|
||||
not read from a git repository. Zuul supports two kinds of reconfigurations.
|
||||
|
||||
The full reconfiguration refetches and reloads the configuration of
|
||||
all tenants. To do so, run ``zuul-scheduler full-reconfigure``. For
|
||||
example this can be used to fix eventual configuration inconsistencies
|
||||
after connection problems to Gerrit/Github.
|
||||
|
||||
The smart reconfiguration reloads only the tenants that changed their
|
||||
configuration in the tenant config file. To do so, run
|
||||
``zuul-scheduler smart-reconfigure``. In multi tenant systems this can
|
||||
be much faster than the full reconfiguration so it is recommended to
|
||||
use the smart reconfiguration after changing the tenant configuration
|
||||
file.
|
||||
|
||||
|
||||
Merger
|
||||
------
|
||||
|
||||
Operation
|
||||
~~~~~~~~~
|
||||
|
||||
To start the merger, run ``zuul-merger``.
|
||||
|
||||
In order to stop the merger and under normal circumstances it is
|
||||
best to pause and wait for all currently running tasks to finish
|
||||
before stopping it. To do so run ``zuul-merger pause``.
|
||||
|
||||
To stop the merger immediately, run ``zuul-merger stop``.
|
||||
|
||||
Executor
|
||||
--------
|
||||
|
||||
Operation
|
||||
~~~~~~~~~
|
||||
|
||||
To start the executor, run ``zuul-executor``.
|
||||
|
||||
There are several commands which can be run to control the executor's
|
||||
behavior once it is running.
|
||||
|
||||
To pause the executor and prevent it from running new jobs you can
|
||||
run ``zuul-executor pause``.
|
||||
|
||||
To cause the executor to stop accepting new jobs and exit when all running
|
||||
jobs have finished you can run ``zuul-executor graceful``. Under most
|
||||
circumstances this will be the best way to stop Zuul.
|
||||
|
||||
To stop the executor immediately, run ``zuul-executor stop``. Jobs that were
|
||||
running on the stopped executor will be rescheduled on other executors.
|
||||
|
||||
The executor normally responds to a ``SIGTERM`` signal in the same way
|
||||
as the ``graceful`` command, however you can change this behavior to match
|
||||
``stop`` with the :attr:`executor.sigterm_method` setting.
|
||||
|
||||
To enable or disable running Ansible in verbose mode (with the
|
||||
``-vvv`` argument to ansible-playbook) run ``zuul-executor verbose``
|
||||
and ``zuul-executor unverbose``.
|
||||
|
||||
.. _ansible-and-python-3:
|
||||
|
||||
Ansible and Python 3
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
As noted above, the executor runs Ansible playbooks against the remote
|
||||
node(s) allocated for the job. Since part of executing playbooks on
|
||||
remote hosts is running Python scripts on them, Ansible needs to know
|
||||
what Python interpreter to use on the remote host. With older
|
||||
distributions, ``/usr/bin/python2`` was a generally sensible choice.
|
||||
However, over time a heterogeneous Python ecosystem has evolved where
|
||||
older distributions may only provide Python 2, most provide a mixed
|
||||
2/3 environment and newer distributions may only provide Python 3 (and
|
||||
then others like RHEL8 may even have separate "system" Python versions
|
||||
to add to confusion!).
|
||||
|
||||
Ansible's ``ansible_python_interpreter`` variable configures the path
|
||||
to the remote Python interpreter to use during playbook execution.
|
||||
This value is set by Zuul from the ``python-path`` specified for the
|
||||
node by Nodepool; see the `nodepool configuration documentation
|
||||
<https://zuul-ci.org/docs/nodepool/configuration.html>`__.
|
||||
|
||||
This defaults to ``auto``, where Ansible will automatically discover
|
||||
the interpreter available on the remote host. However, this setting
|
||||
only became available in Ansible >=2.8, so Zuul will translate
|
||||
``auto`` into the old default of ``/usr/bin/python2`` when configured
|
||||
to use older Ansible versions.
|
||||
|
||||
Thus for modern Python 3-only hosts no further configuration is needed
|
||||
when using Ansible >=2.8 (e.g. Fedora, Bionic onwards). If using
|
||||
earlier Ansible versions you may need to explicitly set the
|
||||
``python-path`` if ``/usr/bin/python2`` is not available on the node.
|
||||
|
||||
Ansible roles/modules which include Python code are generally Python 3
|
||||
safe now, but there is still a small possibility of incompatibility.
|
||||
See also the Ansible `Python 3 support page
|
||||
<https://docs.ansible.com/ansible/latest/reference_appendices/python_3_support.html>`__.
|
||||
|
||||
|
||||
Web Server
|
||||
----------
|
||||
|
||||
Operation
|
||||
~~~~~~~~~
|
||||
|
||||
To start the web server, run ``zuul-web``. To stop it, kill the
|
||||
PID which was saved in the pidfile specified in the configuration.
|
||||
|
||||
Finger Gateway
|
||||
--------------
|
||||
|
||||
|
||||
Operation
|
||||
~~~~~~~~~
|
||||
|
||||
To start the finger gateway, run ``zuul-fingergw``. To stop it, kill the
|
||||
PID which was saved in the pidfile specified in the configuration.
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
:title: Configuration Syntax
|
||||
:title: Project Configuration
|
||||
|
||||
.. _project-config:
|
||||
.. _project-configuration:
|
||||
|
||||
Configuration Syntax
|
||||
====================
|
||||
Project Configuration
|
||||
=====================
|
||||
|
||||
The following sections describe the main part of Zuul's configuration.
|
||||
All of what follows is found within files inside of the repositories
|
||||
|
|
@ -73,6 +73,66 @@ regular expression.
|
|||
Zuul uses the `RE2 library <https://github.com/google/re2/wiki/Syntax>`_
|
||||
which has a restricted regular expression syntax compared to PCRE.
|
||||
|
||||
.. _encryption:
|
||||
|
||||
Encryption
|
||||
----------
|
||||
|
||||
Zuul supports storing encrypted data directly in the git repositories
|
||||
of projects it operates on. If you have a job which requires private
|
||||
information in order to run (e.g., credentials to interact with a
|
||||
third-party service) those credentials can be stored along with the
|
||||
job definition.
|
||||
|
||||
Each project in Zuul has its own automatically generated RSA keypair
|
||||
which can be used by anyone to encrypt a secret and only Zuul is able
|
||||
to decrypt it. Zuul serves each project's public key using its
|
||||
build-in webserver. They can be fetched at the path
|
||||
``/api/tenant/<tenant>/key/<project>.pub`` where ``<project>`` is the
|
||||
canonical name of a project and ``<tenant>`` is the name of a tenant
|
||||
with that project.
|
||||
|
||||
Zuul currently supports one encryption scheme, PKCS#1 with OAEP, which
|
||||
can not store secrets longer than the 3760 bits (derived from the key
|
||||
length of 4096 bits minus 336 bits of overhead). The padding used by
|
||||
this scheme ensures that someone examining the encrypted data can not
|
||||
determine the length of the plaintext version of the data, except to
|
||||
know that it is not longer than 3760 bits (or some multiple thereof).
|
||||
|
||||
In the config files themselves, Zuul uses an extensible method of
|
||||
specifying the encryption scheme used for a secret so that other
|
||||
schemes may be added later. To specify a secret, use the
|
||||
``!encrypted/pkcs1-oaep`` YAML tag along with the base64 encoded
|
||||
value. For example:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
- secret:
|
||||
name: test_secret
|
||||
data:
|
||||
password: !encrypted/pkcs1-oaep |
|
||||
BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi
|
||||
...
|
||||
|
||||
To support secrets longer than 3760 bits, the value after the
|
||||
encryption tag may be a list rather than a scalar. For example:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
- secret:
|
||||
name: long_secret
|
||||
data:
|
||||
password: !encrypted/pkcs1-oaep
|
||||
- er1UXNOD3OqtsRJaP0Wvaqiqx0ZY2zzRt6V9vqIsRaz1R5C4/AEtIad/DERZHwk3Nk+KV
|
||||
...
|
||||
- HdWDS9lCBaBJnhMsm/O9tpzCq+GKRELpRzUwVgU5k822uBwhZemeSrUOLQ8hQ7q/vVHln
|
||||
...
|
||||
|
||||
The `zuul-client utility <https://zuul-ci.org/docs/zuul-client/>`_ provides a
|
||||
simple way to encrypt secrets for a Zuul project:
|
||||
|
||||
.. program-output:: zuul-client encrypt --help
|
||||
|
||||
.. _configuration-items:
|
||||
|
||||
Configuration Items
|
||||
|
|
@ -103,11 +163,11 @@ the YAML files:
|
|||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
pipeline_def
|
||||
job_def
|
||||
project_def
|
||||
queue_def
|
||||
secret_def
|
||||
nodeset_def
|
||||
semaphore_def
|
||||
pragma_def
|
||||
config/pipeline
|
||||
config/job
|
||||
config/project
|
||||
config/queue
|
||||
config/secret
|
||||
config/nodeset
|
||||
config/semaphore
|
||||
config/pragma
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
Admin Reference
|
||||
===============
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 3
|
||||
|
||||
tenants
|
||||
database
|
||||
connections
|
||||
drivers/index
|
||||
client
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
:title: Connection Configuration
|
||||
|
||||
.. _connection-config:
|
||||
|
||||
Connection Configuration
|
||||
========================
|
||||
|
||||
Most of Zuul's configuration is contained in the git repositories upon
|
||||
which Zuul operates, however, some configuration outside of git
|
||||
repositories is still required to bootstrap the system. This includes
|
||||
information on connections between Zuul and other systems, as well as
|
||||
identifying the projects Zuul uses.
|
||||
|
||||
.. _connections:
|
||||
|
||||
Connections
|
||||
-----------
|
||||
|
||||
In order to interact with external systems, Zuul must have a
|
||||
*connection* to that system configured. Zuul includes a number of
|
||||
:ref:`drivers <drivers>`, each of which implements the functionality
|
||||
necessary to connect to a system. Each connection in Zuul is
|
||||
associated with a driver.
|
||||
|
||||
To configure a connection in Zuul, select a unique name for the
|
||||
connection and add a section to ``zuul.conf`` with the form
|
||||
``[connection NAME]``. For example, a connection to a gerrit server
|
||||
may appear as:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[connection mygerritserver]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
|
||||
Zuul needs to use a single connection to look up information about
|
||||
changes hosted by a given system. When it looks up changes, it will
|
||||
do so using the first connection it finds that matches the server name
|
||||
it's looking for. It's generally best to use only a single connection
|
||||
for a given server, however, if you need more than one (for example,
|
||||
to satisfy unique reporting requirements) be sure to list the primary
|
||||
connection first as that is what Zuul will use to look up all changes
|
||||
for that server.
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
:title: Database Configuration
|
||||
|
||||
.. _database:
|
||||
|
||||
Database Configuration
|
||||
======================
|
||||
|
||||
The database configuration is located in the ``database`` section of
|
||||
``zuul.conf``:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[database]
|
||||
dburi=<URI>
|
||||
|
||||
The following options can be defined in this section.
|
||||
|
||||
.. attr:: database
|
||||
|
||||
.. attr:: dburi
|
||||
:required:
|
||||
|
||||
Database connection information in the form of a URI understood
|
||||
by SQLAlchemy. See `The SQLAlchemy manual
|
||||
<https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>`_
|
||||
for more information.
|
||||
|
||||
The driver will automatically set up the database creating and managing
|
||||
the necessary tables. Therefore the provided user should have sufficient
|
||||
permissions to manage the database. For example:
|
||||
|
||||
.. code-block:: sql
|
||||
|
||||
GRANT ALL ON my_database TO 'my_user'@'%';
|
||||
|
||||
.. attr:: pool_recycle
|
||||
:default: 1
|
||||
|
||||
Tune the pool_recycle value. See `The SQLAlchemy manual on pooling
|
||||
<http://docs.sqlalchemy.org/en/latest/core/pooling.html#setting-pool-recycle>`_
|
||||
for more information.
|
||||
|
||||
.. attr:: table_prefix
|
||||
:default: ''
|
||||
|
||||
The string to prefix the table names. This makes it possible to run
|
||||
several zuul deployments against the same database. This can be useful
|
||||
if you rely on external databases which are not under your control.
|
||||
The default is to have no prefix.
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
Reference
|
||||
=========
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
monitoring
|
||||
web
|
||||
developer/index
|
||||
governance
|
||||
vulnerabilities
|
||||
releasenotes
|
||||
glossary
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
User Reference
|
||||
==============
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 3
|
||||
|
||||
config
|
||||
jobs
|
||||
|
|
@ -7,5 +7,5 @@ REST API
|
|||
|
||||
This page is a work-in-progress as the OpenAPI spec is not yet complete.
|
||||
|
||||
.. openapi:: ../../../web/public/openapi.yaml
|
||||
.. openapi:: ../../web/public/openapi.yaml
|
||||
:examples:
|
||||
|
|
@ -337,8 +337,7 @@ configuration. Some examples of tenant definitions are:
|
|||
privileged action.
|
||||
|
||||
More information on tenant-scoped actions can be found in
|
||||
:ref:`tenant-scoped-rest-api`.
|
||||
|
||||
:ref:`authentication`.
|
||||
|
||||
.. attr:: authentication-realm
|
||||
|
||||
|
|
@ -371,8 +370,7 @@ The protected actions available at tenant level are **autohold**, **enqueue**,
|
|||
|
||||
Rules can be overridden by the ``zuul.admin`` claim in a token if if matches
|
||||
an authenticator configuration where `allow_authz_override` is set to true.
|
||||
See :ref:`Zuul web server's configuration <web-server-tenant-scoped-api>` for
|
||||
more details.
|
||||
See :ref:`authentication` for more details.
|
||||
|
||||
Below are some examples of how access rules can be defined:
|
||||
|
||||
|
|
@ -437,8 +435,7 @@ Below are some examples of how access rules can be defined:
|
|||
|
||||
The special ``zuul_uid`` claim refers to the ``uid_claim`` setting in an
|
||||
authenticator's configuration. By default it refers to the ``sub`` claim
|
||||
of a token. For more details see the :ref:`configuration section
|
||||
<web-server-tenant-scoped-api>` for Zuul web server.
|
||||
of a token. For more details see the :ref:`authentication`.
|
||||
|
||||
Under the above example, the following token would match rules
|
||||
``affiliate_or_admin`` and ``alice_or_bob``:
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
Admin Tutorials
|
||||
===============
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
quick-start
|
||||
keycloak
|
||||
|
|
@ -3,7 +3,7 @@ Keycloak Tutorial
|
|||
|
||||
Zuul supports an authenticated API accessible via its web app which
|
||||
can be used to perform some administrative actions. To see this in
|
||||
action, first run the :ref:`quick_start` and then follow the steps in
|
||||
action, first run the :ref:`quick-start` and then follow the steps in
|
||||
this tutorial to add a Keycloak server.
|
||||
|
||||
Zuul supports any identity provider that can supply a JWT using OpenID
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
.. _quick_start:
|
||||
.. _quick-start:
|
||||
|
||||
Quick-Start Installation and Tutorial
|
||||
=====================================
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
User Tutorials
|
||||
==============
|
||||
|
||||
The User tutorials are currently a work in progress. Currently, only admin
|
||||
related tutorials are available.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
admin
|
||||
|
||||
Loading…
Reference in New Issue