diff --git a/.gitignore b/.gitignore index e76a1bd32d..d7e2fac657 100644 --- a/.gitignore +++ b/.gitignore @@ -3,14 +3,17 @@ *.egg-info *.pyc .idea +.mypy_cache .test .testrepository .tox .venv +.coverage AUTHORS build/* ChangeLog -config doc/build/* zuul/versioninfo dist/ +cover/ +htmlcov/ diff --git a/.testr.conf b/.testr.conf index 8ef6689074..7e8d028994 100644 --- a/.testr.conf +++ b/.testr.conf @@ -1,4 +1,4 @@ [DEFAULT] -test_command=OS_LOG_LEVEL=${OS_LOG_LEVEL:-INFO} OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} OS_LOG_DEFAULTS=${OS_LOG_DEFAULTS:-""} ${PYTHON:-python} -m subunit.run discover -t ./ tests $LISTOPT $IDOPTION +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} OS_LOG_DEFAULTS=${OS_LOG_DEFAULTS:-""} ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./tests/unit} $LISTOPT $IDOPTION test_id_option=--load-list $IDFILE test_list_option=--list diff --git a/.zuul.yaml b/.zuul.yaml index 9f0382b79f..c820c8e9c2 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,14 +1,64 @@ +- nodeset: + name: zuul-functional-temp-master + nodes: + - name: controller + label: ubuntu-xenial + - name: node1 + label: ubuntu-xenial + - name: node2 + label: ubuntu-xenial + groups: + - name: node + nodes: + - node1 + - node2 + +- job: + name: zuul-stream-functional + parent: multinode + nodeset: zuul-functional-temp-master + pre-run: playbooks/zuul-stream/pre.yaml + run: playbooks/zuul-stream/functional.yaml + post-run: + - playbooks/zuul-stream/post.yaml + - playbooks/zuul-stream/post-ara.yaml + required-projects: + - openstack/ara + files: + - zuul/ansible/callback/.* + - playbooks/zuul-stream/.* + - project: name: openstack-infra/zuul check: jobs: - - tox-docs - - tox-cover: - voting: false + - build-openstack-sphinx-docs: + irrelevant-files: + - zuul/cmd/migrate.py + - playbooks/zuul-migrate/.* + vars: + sphinx_python: python3 - tox-pep8 - - tox-py27 + - tox-py35: + irrelevant-files: + - zuul/cmd/migrate.py + - playbooks/zuul-migrate/.* + - zuul-stream-functional gate: jobs: - - tox-docs + - build-openstack-sphinx-docs: + irrelevant-files: + - zuul/cmd/migrate.py + - playbooks/zuul-migrate/.* + vars: + sphinx_python: python3 - tox-pep8 - - tox-py27 + - tox-py35: + irrelevant-files: + - zuul/cmd/migrate.py + - playbooks/zuul-migrate/.* + - zuul-stream-functional + post: + jobs: + - publish-openstack-sphinx-docs-infra-python3 + - publish-openstack-python-branch-tarball diff --git a/NEWS.rst b/NEWS.rst index 28a6ec2646..5fef40af82 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -12,11 +12,6 @@ Since 2.0.0: the Zuul server in smaller deployments. Several configuration options have moved from the ``zuul`` section to ``merger``. -* Gerrit label names must now be listed in your layout.yaml exactly as - they appear in Gerrit. This means case and special characters must - match. This change was made to accomodate Gerrit 2.13 which needs the - strings to match for changes to be successfully submitted. - Since 1.3.0: * The Jenkins launcher is replaced with Gearman launcher. An internal diff --git a/README.rst b/README.rst index 90e00a56f3..8d00665306 100644 --- a/README.rst +++ b/README.rst @@ -3,6 +3,21 @@ Zuul Zuul is a project gating system developed for the OpenStack Project. +We are currently engaged in a significant development effort in +preparation for the third major version of Zuul. We call this effort +`Zuul v3`_ and it is described in more detail below. + +The latest documentation for Zuul v3 is published at: +https://docs.openstack.org/infra/zuul/feature/zuulv3/ + +If you are looking for the Edge routing service named Zuul that is +related to Netflix, it can be found here: +https://github.com/Netflix/zuul + +If you are looking for the Javascript testing tool named Zuul, it +can be found here: +https://github.com/defunctzombie/zuul + Contributing ------------ @@ -25,3 +40,131 @@ that links to your launchpad account). Example:: # Do your commits $ git review # Enter your username if prompted + +Zuul v3 +------- + +The Zuul v3 effort involves significant changes to Zuul, and its +companion program, Nodepool. The intent is for Zuul to become more +generally useful outside of the OpenStack community. This is the best +way to get started with this effort: + +1) Read the Zuul v3 spec: http://specs.openstack.org/openstack-infra/infra-specs/specs/zuulv3.html + + We use specification documents like this to describe large efforts + where we want to make sure that all the participants are in + agreement about what will happen and generally how before starting + development. These specs should contain enough information for + people to evaluate the proposal generally, and sometimes include + specific details that need to be agreed upon in advance. They are + living documents which can change as work gets underway. However, + every change or detail does not need to be reflected in the spec -- + most work is simply done with patches (and revised if necessary in + code review). + +2) Read the Nodepool build-workers spec: http://specs.openstack.org/openstack-infra/infra-specs/specs/nodepool-zookeeper-workers.html + +3) Review any proposed updates to these specs: https://review.openstack.org/#/q/status:open+project:openstack-infra/infra-specs+topic:zuulv3 + + Some of the information in the specs may be effectively superceded + by changes here, which are still undergoing review. + +4) Read developer documentation on the internal data model and testing: http://docs.openstack.org/infra/zuul/feature/zuulv3/developer.html + + The general philosophy for Zuul tests is to perform functional + testing of either the individual component or the entire end-to-end + system with external systems (such as Gerrit) replaced with fakes. + Before adding additional unit tests with a narrower focus, consider + whether they add value to this system or are merely duplicative of + functional tests. + +5) Review open changes: https://review.openstack.org/#/q/status:open+branch:feature/zuulv3 + + We find that the most valuable code reviews are ones that spot + problems with the proposed change, or raise questions about how + that might affect other systems or subsequent work. It is also a + great way to stay involved as a team in work performed by others + (for instance, by observing and asking questions about development + while it is in progress). We try not to sweat the small things and + don't worry too much about style suggestions or other nitpicky + things (unless they are relevant -- for instance, a -1 vote on a + change that introduces a yaml change out of character with existing + conventions is useful because it makes the system more + user-friendly; a -1 vote on a change which uses a sub-optimal line + breaking strategy is probably not the best use of anyone's time). + +6) Join #zuul on Freenode. Let others (especially jeblair who is + trying to coordinate and prioritize work) know what you would like + to work on. + +7) Check storyboard for status of current work items: https://storyboard.openstack.org/#!/board/41 + + Work items tagged with ``low-hanging-fruit`` are tasks that have + been identified as not requiring an expansive knowledge of the + system. They may still require either some knowledge or + investigation into a specific area, but should be suitable for a + developer who is becoming acquainted with the system. Those items + can be found at: + https://storyboard.openstack.org/#!/story/list?tags=low-hanging-fruit&tags=zuulv3 + +Once you are up to speed on those items, it will be helpful to know +the following: + +* Zuul v3 includes some substantial changes to Zuul, and in order to + implement them quickly and simultaneously, we temporarily disabled + most of the test suite. That test suite still has relevance, but + tests are likely to need updating individually, with reasons ranging + from something simple such as a test-framework method changing its + name, to more substantial issues, such as a feature being removed as + part of the v3 work. Each test will need to be evaluated + individually. Feel free to, at any time, claim a test name in this + story and work on re-enabling it: + https://storyboard.openstack.org/#!/story/2000773 + +* Because of the importance of external systems, as well as the number + of internal Zuul components, actually running Zuul in a development + mode quickly becomes unweildy (imagine uploading changes to Gerrit + repeatedly while altering Zuul source code). Instead, the best way + to develop with Zuul is in fact to write a functional test. + Construct a test to fully simulate the series of events you want to + see, then run it in the foreground. For example:: + + .tox/py27/bin/python -m testtools.run tests.unit.test_scheduler.TestScheduler.test_jobs_executed + + See TESTING.rst for more information. + +* There are many occasions, when working on sweeping changes to Zuul + v3, we left notes for future work items in the code marked with + "TODOv3". These represent potentially serious missing functionality + or other issues which must be resolved before an initial v3 release + (unlike a more conventional TODO note, these really can not be left + indefinitely). These present an opportunity to identify work items + not otherwise tracked. The names associated with TODO or TODOv3 + items do not mean that only that person can address them -- they + simply reflect who to ask to explain the item in more detail if it + is too cryptic. In your own work, feel free to leave TODOv3 notes + if a change would otherwise become too large or unweildy. + +Python Version Support +---------------------- + +Zuul v3 requires Python 3. It does not support Python 2. + +As Ansible is used for the execution of jobs, it's important to note that +while Ansible does support Python 3, not all of Ansible's modules do. Zuul +currently sets ``ansible_python_interpreter`` to python2 so that remote +content will be executed with Python2. + +Roadmap +------- + +* Begin using Zuul v3 to run jobs for Zuul itself +* Implement a shim to translate Zuul v2 demand into Nodepool Zookeeper + launcher requests +* Begin using Zookeeper based Nodepool launchers with Zuul v2.5 in + OpenStack Infra +* Move OpenStack Infra to use Zuul v3 +* Implement Github support +* Begin using Zuul v3 to run tests on Ansible repos +* Implement support in Nodepool for non-OpenStack clouds +* Add native container support to Zuul / Nodepool diff --git a/TESTING.rst b/TESTING.rst index 56f2fbb5e3..0786ebf408 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -17,6 +17,16 @@ More information on pip here: http://www.pip-installer.org/en/latest/ pip install tox +As of zuul v3, a running zookeeper is required to execute tests. + +*Install zookeeper*:: + + [apt-get | yum] install zookeeperd + +*Start zookeeper*:: + + service zookeeper start + Run The Tests ------------- @@ -54,12 +64,12 @@ To run individual tests with tox:: For example, to *run the basic Zuul test*:: - tox -e py27 -- tests.test_scheduler.TestScheduler.test_jobs_launched + tox -e py27 -- tests.unit.test_scheduler.TestScheduler.test_jobs_executed To *run one test in the foreground* (after previously having run tox to set up the virtualenv):: - .tox/py27/bin/python -m testtools.run tests.test_scheduler.TestScheduler.test_jobs_launched + .tox/py27/bin/python -m testtools.run tests.unit.test_scheduler.TestScheduler.test_jobs_executed List Failing Tests ------------------ diff --git a/bindep.txt b/bindep.txt index 32c750ab12..3dcc3e7cd5 100644 --- a/bindep.txt +++ b/bindep.txt @@ -4,3 +4,16 @@ mysql-client [test] mysql-server [test] libjpeg-dev [test] +openssl [test] +zookeeperd [platform:dpkg] +build-essential [platform:dpkg] +gcc [platform:rpm] +graphviz [doc] +libssl-dev [platform:dpkg] +openssl-devel [platform:rpm] +libffi-dev [platform:dpkg] +libffi-devel [platform:rpm] +python-dev [platform:dpkg] +python-devel [platform:rpm] +bubblewrap [platform:rpm] +redhat-rpm-config [platform:rpm] diff --git a/doc/source/admin/client.rst b/doc/source/admin/client.rst new file mode 100644 index 0000000000..8ee6b6fbbd --- /dev/null +++ b/doc/source/admin/client.rst @@ -0,0 +1,115 @@ +:title: Zuul Client + +Zuul Client +=========== + +Zuul includes a simple command line client that may be used by +administrators to affect Zuul's behavior while running. It must be +run on a host that has access to the Gearman server (e.g., locally on +the Zuul host). + +Configuration +------------- + +The client uses the same zuul.conf file as the server, and will look +for it in the same locations if not specified on the command line. + +Usage +----- +The general options that apply to all subcommands are: + +.. program-output:: zuul --help + +The following subcommands are supported: + +Autohold +^^^^^^^^ +.. program-output:: zuul autohold --help + +Example:: + + zuul autohold --tenant openstack --project example_project --job example_job --reason "reason text" --count 1 + +Enqueue +^^^^^^^ +.. program-output:: zuul enqueue --help + +Example:: + + zuul enqueue --tenant openstack --trigger gerrit --pipeline check --project example_project --change 12345,1 + +Note that the format of change id is ,. + +Enqueue-ref +^^^^^^^^^^^ + +.. program-output:: zuul enqueue-ref --help + +This command is provided to manually simulate a trigger from an +external source. It can be useful for testing or replaying a trigger +that is difficult or impossible to recreate at the source. The +arguments to ``enqueue-ref`` will vary depending on the source and +type of trigger. Some familiarity with the arguments emitted by +``gerrit`` `update hooks +`__ +such as ``patchset-created`` and ``ref-updated`` is recommended. Some +examples of common operations are provided below. + +Manual enqueue examples +*********************** + +It is common to have a ``release`` pipeline that listens for new tags +coming from ``gerrit`` and performs a range of code packaging jobs. +If there is an unexpected issue in the release jobs, the same tag can +not be recreated in ``gerrit`` and the user must either tag a new +release or request a manual re-triggering of the jobs. To re-trigger +the jobs, pass the failed tag as the ``ref`` argument and set +``newrev`` to the change associated with the tag in the project +repository (i.e. what you see from ``git show X.Y.Z``):: + + zuul enqueue-ref --tenant openstack --trigger gerrit --pipeline release --project openstack/example_project --ref refs/tags/X.Y.Z --newrev abc123... + +The command can also be used asynchronosly trigger a job in a +``periodic`` pipeline that would usually be run at a specific time by +the ``timer`` driver. For example, the following command would +trigger the ``periodic`` jobs against the current ``master`` branch +top-of-tree for a project:: + + zuul enqueue-ref --tenant openstack --trigger timer --pipeline periodic --project openstack/example_project --ref refs/heads/master + +Another common pipeline is a ``post`` queue listening for ``gerrit`` +merge results. Triggering here is slightly more complicated as you +wish to recreate the full ``ref-updated`` event from ``gerrit``. For +a new commit on ``master``, the gerrit ``ref-updated`` trigger +expresses "reset ``refs/heads/master`` for the project from ``oldrev`` +to ``newrev``" (``newrev`` being the committed change). Thus to +replay the event, you could ``git log`` in the project and take the +current ``HEAD`` and the prior change, then enqueue the event:: + + NEW_REF=$(git rev-parse HEAD) + OLD_REF=$(git rev-parse HEAD~1) + + zuul enqueue-ref --tenant openstack --trigger gerrit --pipeline post --project openstack/example_project --ref refs/heads/master --newrev $NEW_REF --oldrev $OLD_REF + +Note that zero values for ``oldrev`` and ``newrev`` can indicate +branch creation and deletion; the source code is the best reference +for these more advanced operations. + + +Promote +^^^^^^^ +.. program-output:: zuul promote --help + +Example:: + + zuul promote --tenant openstack --pipeline check --changes 12345,1 13336,3 + +Note that the format of changes id is ,. + +Show +^^^^ +.. program-output:: zuul show --help + +Example:: + + zuul show running-jobs diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst new file mode 100644 index 0000000000..d6b0984d24 --- /dev/null +++ b/doc/source/admin/components.rst @@ -0,0 +1,739 @@ +:title: Components + +.. _components: + +Components +========== + +Zuul is a distributed system consisting of several components, each of +which is described below. + +.. graphviz:: + :align: center + + graph { + node [shape=box] + Gearman [shape=ellipse] + Gerrit [fontcolor=grey] + Zookeeper [shape=ellipse] + Nodepool + GitHub [fontcolor=grey] + + Merger -- Gearman + Executor -- Gearman + Web -- Gearman + + Gearman -- Scheduler; + Scheduler -- Gerrit; + Scheduler -- Zookeeper; + Zookeeper -- Nodepool; + Scheduler -- GitHub; + } + +Each of the Zuul processes may run on the same host, or different +hosts. Within Zuul, the components communicate with the scheduler via +the Gearman protocol, so each Zuul component needs to be able to +connect to the host running the Gearman server (the scheduler has a +built-in Gearman server which is recommended) on the Gearman port -- +TCP port 4730 by default. + +The Zuul scheduler communicates with Nodepool via the ZooKeeper +protocol. Nodepool requires an external ZooKeeper cluster, and the +Zuul scheduler needs to be able to connect to the hosts in that +cluster on TCP port 2181. + +Both the Nodepool launchers and Zuul executors need to be able to +communicate with the hosts which nodepool provides. If these are on +private networks, the Executors will need to be able to route traffic +to them. + +If statsd is enabled, every service needs to be able to emit data to +statsd. Statsd can be configured to run on each host and forward +data, or services may emit to a centralized statsd collector. Statsd +listens on UDP port 8125 by default. + +All Zuul processes read the ``/etc/zuul/zuul.conf`` file (an alternate +location may be supplied on the command line) which uses an INI file +syntax. Each component may have its own configuration file, though +you may find it simpler to use the same file for all components. + +An example ``zuul.conf``: + +.. code-block:: ini + + [gearman] + server=localhost + + [gearman_server] + start=true + log_config=/etc/zuul/gearman-logging.yaml + + [zookeeper] + hosts=zk1.example.com,zk2.example.com,zk3.example.com + + [webapp] + status_url=https://zuul.example.com/status + + [scheduler] + log_config=/etc/zuul/scheduler-logging.yaml + +A minimal Zuul system may consist of a :ref:`scheduler` and +:ref:`executor` both running on the same host. Larger installations +should consider running multiple executors, each on a dedicated host, +and running mergers on dedicated hosts as well. + +Common +------ + +The following applies to all Zuul components. + +Configuration +~~~~~~~~~~~~~ + +The following sections of ``zuul.conf`` are used by all Zuul components: + + +.. attr:: gearman + + Client connection information for Gearman. + + .. attr:: server + :required: + + Hostname or IP address of the Gearman server. + + .. attr:: port + :default: 4730 + + Port on which the Gearman server is listening. + + .. attr:: ssl_ca + + An openssl file containing a set of concatenated “certification + authority” certificates in PEM formet. + + .. attr:: ssl_cert + + An openssl file containing the client public certificate in PEM format. + + .. attr:: ssl_key + + An openssl file containing the client private key in PEM format. + +.. attr:: statsd + + Information about the optional statsd server. If the ``statsd`` + python module is installed and this section is configured, + statistics will be reported to statsd. See :ref:`statsd` for more + information. + + .. attr:: server + + Hostname or IP address of the statsd server. + + .. attr:: port + :default: 8125 + + The UDP port on which the statsd server is listening. + + .. attr:: prefix + + If present, this will be prefixed to all of the keys before + transmitting to the statsd server. + +.. NOTE: this is a white lie at this point, since only the scheduler + uses this, however, we expect other components to use it later, so + it's reasonable for admins to plan for this now. + +.. attr:: zookeeper + + Client connection information for ZooKeeper + + .. attr:: hosts + :required: + + A list of zookeeper hosts for Zuul to use when communicating + with Nodepool. + + .. attr:: session_timeout + :default: 10.0 + + The ZooKeeper session timeout, in seconds. + + +.. _scheduler: + +Scheduler +--------- + +The scheduler is the primary component of Zuul. The scheduler is not +a scalable component; one, and only one, scheduler must be running at +all times for Zuul to be operational. It receives events from any +connections to remote systems which have been configured, enqueues +items into pipelines, distributes jobs to executors, and reports +results. + +The scheduler includes a Gearman server which is used to communicate +with other components of Zuul. It is possible to use an external +Gearman server, but the built-in server is well-tested and +recommended. If the built-in server is used, other Zuul hosts will +need to be able to connect to the scheduler on the Gearman port, TCP +port 4730. It is also strongly recommended to use SSL certs with +Gearman, as secrets are transferred from the scheduler to executors +over this link. + +The scheduler must be able to connect to the ZooKeeper cluster used by +Nodepool in order to request nodes. It does not need to connect +directly to the nodes themselves, however -- that function is handled +by the Executors. + +It must also be able to connect to any services for which connections +are configured (Gerrit, GitHub, etc). + +Configuration +~~~~~~~~~~~~~ + +The following sections of ``zuul.conf`` are used by the scheduler: + + +.. attr:: gearman_server + + The builtin gearman server. Zuul can fork a gearman process from + itself rather than connecting to an external one. + + .. attr:: start + :default: false + + Whether to start the internal Gearman server. + + .. attr:: listen_address + :default: all addresses + + IP address or domain name on which to listen. + + .. attr:: port + :default: 4730 + + TCP port on which to listen. + + .. attr:: log_config + + Path to log config file for internal Gearman server. + + .. attr:: ssl_ca + + An openssl file containing a set of concatenated “certification + authority” certificates in PEM formet. + + .. attr:: ssl_cert + + An openssl file containing the server public certificate in PEM + format. + + .. attr:: ssl_key + + An openssl file containing the server private key in PEM format. + +.. attr:: webapp + + .. attr:: listen_address + :default: all addresses + + IP address or domain name on which to listen. + + .. attr:: port + :default: 8001 + + Port on which the webapp is listening. + + .. attr:: status_expiry + :default: 1 + + Zuul will cache the status.json file for this many seconds. + + .. attr:: status_url + + URL that will be posted in Zuul comments made to changes when + starting jobs for a change. + + .. TODO: is this effectively required? + +.. attr:: scheduler + + .. attr:: command_socket + :default: /var/lib/zuul/scheduler.socket + + Path to command socket file for the scheduler process. + + .. attr:: tenant_config + :required: + + Path to :ref:`tenant-config` file. + + .. attr:: log_config + + Path to log config file. + + .. attr:: pidfile + :default: /var/run/zuul-schedurecr/zuul-scheduler.pid + + Path to PID lock file. + + .. attr:: state_dir + :default: /var/lib/zuul + + Path to directory in which Zuul should save its state. + +Operation +~~~~~~~~~ + +To start the scheduler, run ``zuul-scheduler``. To stop it, kill the +PID which was saved in the pidfile specified in the configuration. + +Most of Zuul's configuration is automatically updated as changes to +the repositories which contain it are merged. However, Zuul must be +explicitly notified of changes to the tenant config file, since it is +not read from a git repository. To do so, send the scheduler PID +(saved in the pidfile specified in the configuration) a `SIGHUP` +signal. + +Merger +------ + +Mergers are an optional Zuul service; they are not required for Zuul +to operate, but some high volume sites may benefit from running them. +Zuul performs quite a lot of git operations in the course of its work. +Each change that is to be tested must be speculatively merged with the +current state of its target branch to ensure that it can merge, and to +ensure that the tests that Zuul perform accurately represent the +outcome of merging the change. Because Zuul's configuration is stored +in the git repos it interacts with, and is dynamically evaluated, Zuul +often needs to perform a speculative merge in order to determine +whether it needs to perform any further actions. + +All of these git operations add up, and while Zuul executors can also +perform them, large numbers may impact their ability to run jobs. +Therefore, administrators may wish to run standalone mergers in order +to reduce the load on executors. + +Mergers need to be able to connect to the Gearman server (usually the +scheduler host) as well as any services for which connections are +configured (Gerrit, GitHub, etc). + +Configuration +~~~~~~~~~~~~~ + +The following section of ``zuul.conf`` is used by the merger: + +.. attr:: merger + + .. attr:: command_socket + :default: /var/lib/zuul/merger.socket + + Path to command socket file for the merger process. + + .. attr:: git_dir + + Directory in which Zuul should clone git repositories. + + .. attr:: git_http_low_speed_limit + :default: 1000 + + If the HTTP transfer speed is less then git_http_low_speed_limit for + longer then git_http_low_speed_time, the transfer is aborted. + + Value in bytes, setting to 0 will disable. + + .. attr:: git_http_low_speed_time + :default: 30 + + If the HTTP transfer speed is less then git_http_low_speed_limit for + longer then git_http_low_speed_time, the transfer is aborted. + + Value in seconds, setting to 0 will disable. + + .. attr:: git_user_email + + Value to pass to `git config user.email + `_. + + .. attr:: git_user_name + + Value to pass to `git config user.name + `_. + + .. attr:: log_config + + Path to log config file for the merger process. + + .. attr:: pidfile + :default: /var/run/zuul-merger/zuul-merger.pid + + Path to PID lock file for the merger process. + +Operation +~~~~~~~~~ + +To start the merger, run ``zuul-merger``. To stop it, kill the +PID which was saved in the pidfile specified in the configuration. + +.. _executor: + +Executor +-------- + +Executors are responsible for running jobs. At the start of each job, +an executor prepares an environment in which to run Ansible which +contains all of the git repositories specified by the job with all +dependent changes merged into their appropriate branches. The branch +corresponding to the proposed change will be checked out (in all +projects, if it exists). Any roles specified by the job will also be +present (also with dependent changes merged, if appropriate) and added +to the Ansible role path. The executor also prepares an Ansible +inventory file with all of the nodes requested by the job. + +The executor also contains a merger. This is used by the executor to +prepare the git repositories used by jobs, but is also available to +perform any tasks normally performed by standalone mergers. Because +the executor performs both roles, small Zuul installations may not +need to run standalone mergers. + +Executors need to be able to connect to the Gearman server (usually +the scheduler host), any services for which connections are configured +(Gerrit, GitHub, etc), as well as directly to the hosts which Nodepool +provides. + +Trusted and Untrusted Playbooks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The executor runs playbooks in one of two execution contexts depending +on whether the project containing the playbook is a +:term:`config-project` or an :term:`untrusted-project`. If the +playbook is in a config project, the executor runs the playbook in the +*trusted* execution context, otherwise, it is run in the *untrusted* +execution context. + +Both execution contexts use `bubblewrap`_ [#nullwrap]_ to create a +namespace to ensure that playbook executions are isolated and are unable +to access files outside of a restricted environment. The administrator +may configure additional local directories on the executor to be made +available to the restricted environment. + +The trusted execution context has access to all Ansible features, +including the ability to load custom Ansible modules. Needless to +say, extra scrutiny should be given to code that runs in a trusted +context as it could be used to compromise other jobs running on the +executor, or the executor itself, especially if the administrator has +granted additional access through bubblewrap, or a method of escaping +the restricted environment created by bubblewrap is found. + +Playbooks run in the untrusted execution context are not permitted to +load additional Ansible modules or access files outside of the +restricted environment prepared for them by the executor. In addition +to the bubblewrap environment applied to both execution contexts, in +the untrusted context some standard Ansible modules are replaced with +versions which prohibit some actions, including attempts to access +files outside of the restricted execution context. These redundant +protections are made as part of a defense-in-depth strategy. + +.. _bubblewrap: https://github.com/projectatomic/bubblewrap +.. [#nullwrap] Unless one has set execution_wrapper to nullwrap in the + executor configuration. + +Configuration +~~~~~~~~~~~~~ + +The following sections of ``zuul.conf`` are used by the executor: + +.. attr:: executor + + .. attr:: command_socket + :default: /var/lib/zuul/executor.socket + + Path to command socket file for the executor process. + + .. attr:: finger_port + :default: 7900 + + Port to use for finger log streamer. + + .. attr:: git_dir + :default: /var/lib/zuul/git + + Directory that Zuul should clone local git repositories to. The + executor keeps a local copy of every git repository it works + with to speed operations and perform speculative merging. + + This should be on the same filesystem as + :attr:`executor.job_dir` so that when git repos are cloned into + the job workspaces, they can be hard-linked to the local git + cache. + + .. attr:: job_dir + :default: /tmp + + Directory that Zuul should use to hold temporary job directories. + When each job is run, a new entry will be created under this + directory to hold the configuration and scratch workspace for + that job. It will be deleted at the end of the job (unless the + `--keep-jobdir` command line option is specified). + + This should be on the same filesystem as :attr:`executor.git_dir` + so that when git repos are cloned into the job workspaces, they + can be hard-linked to the local git cache. + + .. attr:: log_config + + Path to log config file for the executor process. + + .. attr:: pidfile + :default: /var/run/zuul-executor/zuul-executor.pid + + Path to PID lock file for the executor process. + + .. attr:: private_key_file + :default: ~/.ssh/id_rsa + + SSH private key file to be used when logging into worker nodes. + + .. _admin_sitewide_variables: + + .. attr:: variables + + Path to an Ansible variables file to supply site-wide variables. + This should be a YAML-formatted file consisting of a single + dictionary. The contents will be made available to all jobs as + Ansible variables. These variables take precedence over all + other forms (job variables and secrets). Care should be taken + when naming these variables to avoid potential collisions with + those used by jobs. Prefixing variable names with a + site-specific identifier is recommended. The default is not to + add any site-wide variables. See the :ref:`User's Guide + ` for more information. + + .. attr:: disk_limit_per_job + :default: 250 + + This integer is the maximum number of megabytes that any one job + is allowed to consume on disk while it is running. If a job's + scratch space has more than this much space consumed, it will be + aborted. + + .. attr:: trusted_ro_paths + + List of paths, separated by ``:`` to read-only bind mount into + trusted bubblewrap contexts. + + .. attr:: trusted_rw_paths + + List of paths, separated by ``:`` to read-write bind mount into + trusted bubblewrap contexts. + + .. attr:: untrusted_ro_paths + + List of paths, separated by ``:`` to read-only bind mount into + untrusted bubblewrap contexts. + + .. attr:: untrusted_rw_paths + + List of paths, separated by ``:`` to read-write bind mount into + untrusted bubblewrap contexts. + + .. attr:: execution_wrapper + :default: bubblewrap + + Name of the execution wrapper to use when executing + `ansible-playbook`. The default, `bubblewrap` is recommended for + all installations. + + There is also a `nullwrap` driver for situations where one wants + to run Zuul without access to bubblewrap or in such a way that + bubblewrap may interfere with the jobs themselves. However, + `nullwrap` is considered unsafe, as `bubblewrap` provides + significant protections against malicious users and accidental + breakage in playbooks. As such, `nullwrap` is not recommended + for use in production. + + This option, and thus, `nullwrap`, may be removed in the future. + `bubblewrap` has become integral to securely operating Zuul. If you + have a valid use case for it, we encourage you to let us know. + + .. attr:: load_multiplier + :default: 2.5 + + When an executor host gets too busy, the system may suffer + timeouts and other ill effects. The executor will stop accepting + more than 1 job at a time until load has lowered below a safe + level. This level is determined by multiplying the number of + CPU's by `load_multiplier`. + + So for example, if the system has 2 CPUs, and load_multiplier + is 2.5, the safe load for the system is 5.00. Any time the + system load average is over 5.00, the executor will quit + accepting multiple jobs at one time. + + The executor will observe system load and determine whether + to accept more jobs every 30 seconds. + + .. attr:: hostname + :default: hostname of the server + + The executor needs to know its hostname under which it is reachable by + zuul-web. Otherwise live console log streaming doesn't work. In most cases + This is automatically detected correctly. But when running in environments + where it cannot determine its hostname correctly this can be overridden + here. + +.. attr:: merger + + .. attr:: git_user_email + + Value to pass to `git config user.email + `_. + + .. attr:: git_user_name + + Value to pass to `git config user.name + `_. + +Operation +~~~~~~~~~ + +To start the executor, run ``zuul-executor``. + +There are several commands which can be run to control the executor's +behavior once it is running. + +To stop the executor immediately, aborting all jobs (they may be +relaunched according to their retry policy), run ``zuul-executor +stop``. + +To request that the executor stop executing new jobs and exit when all +currently running jobs have completed, run ``zuul-executor graceful``. + +To enable or disable running Ansible in verbose mode (with the +``-vvv`` argument to ansible-playbook) run ``zuul-executor verbose`` +and ``zuul-executor unverbose``. + +Web Server +---------- + +The Zuul web server currently acts as a websocket interface to live log +streaming. Eventually, it will serve as the single process handling all +HTTP interactions with Zuul. + +Web servers need to be able to connect to the Gearman server (usually +the scheduler host). If the SQL reporter is used, they need to be +able to connect to the database it reports to in order to support the +dashboard. If a GitHub connection is configured, they need to be +reachable by GitHub so they may receive notifications. + +Configuration +~~~~~~~~~~~~~ + +In addition to the common configuration sections, the following +sections of ``zuul.conf`` are used by the web server: + +.. attr:: web + + .. attr:: listen_address + :default: 127.0.0.1 + + IP address or domain name on which to listen. + + .. attr:: log_config + + Path to log config file for the web server process. + + .. attr:: pidfile + :default: /var/run/zuul-web/zuul-web.pid + + Path to PID lock file for the web server process. + + .. attr:: port + :default: 9000 + + Port to use for web server process. + + .. attr:: websocket_url + + Base URL on which the websocket service is exposed, if different + than the base URL of the web app. + + .. attr:: static_cache_expiry + :default: 3600 + + The Cache-Control max-age response header value for static files served + by the zuul-web. Set to 0 during development to disable Cache-Control. + +Operation +~~~~~~~~~ + +To start the web server, run ``zuul-web``. To stop it, kill the +PID which was saved in the pidfile specified in the configuration. + +Finger Gateway +-------------- + +The Zuul finger gateway listens on the standard finger port (79) for +finger requests specifying a build UUID for which it should stream log +results. The gateway will determine which executor is currently running that +build and query that executor for the log stream. + +This is intended to be used with the standard finger command line client. +For example:: + + finger UUID@zuul.example.com + +The above would stream the logs for the build identified by `UUID`. + +Finger gateway servers need to be able to connect to the Gearman +server (usually the scheduler host), as well as the console streaming +port on the executors (usually 7900). + +Configuration +~~~~~~~~~~~~~ + +In addition to the common configuration sections, the following +sections of ``zuul.conf`` are used by the finger gateway: + +.. attr:: fingergw + + .. attr:: command_socket + :default: /var/lib/zuul/fingergw.socket + + Path to command socket file for the executor process. + + .. attr:: listen_address + :default: all addresses + + IP address or domain name on which to listen. + + .. attr:: log_config + + Path to log config file for the finger gateway process. + + .. attr:: pidfile + :default: /var/run/zuul-fingergw/zuul-fingergw.pid + + Path to PID lock file for the finger gateway process. + + .. attr:: port + :default: 79 + + Port to use for the finger gateway. Note that since command line + finger clients cannot usually specify the port, leaving this set to + the default value is highly recommended. + + .. attr:: user + :default: zuul + + User ID for the zuul-fingergw process. In normal operation as a + daemon, the finger gateway should be started as the ``root`` user, but + it will drop privileges to this user during startup. + +Operation +~~~~~~~~~ + +To start the finger gateway, run ``zuul-fingergw``. To stop it, kill the +PID which was saved in the pidfile specified in the configuration. diff --git a/doc/source/admin/connections.rst b/doc/source/admin/connections.rst new file mode 100644 index 0000000000..b04dbb06a7 --- /dev/null +++ b/doc/source/admin/connections.rst @@ -0,0 +1,71 @@ +:title: Connection Configuration + +.. _connection-config: + +Connection Configuration +======================== + +Most of Zuul's configuration is contained in the git repositories upon +which Zuul operates, however, some configuration outside of git +repositories is still required to bootstrap the system. This includes +information on connections between Zuul and other systems, as well as +identifying the projects Zuul uses. + +.. _connections: + +Connections +----------- + +In order to interact with external systems, Zuul must have a +*connection* to that system configured. Zuul includes a number of +drivers, each of which implements the functionality necessary to +connect to a system. Each connection in Zuul is associated with a +driver. + +To configure a connection in Zuul, select a unique name for the +connection and add a section to ``zuul.conf`` with the form +``[connection NAME]``. For example, a connection to a gerrit server +may appear as: + +.. code-block:: ini + + [connection mygerritserver] + driver=gerrit + server=review.example.com + +Zuul needs to use a single connection to look up information about +changes hosted by a given system. When it looks up changes, it will +do so using the first connection it finds that matches the server name +it's looking for. It's generally best to use only a single connection +for a given server, however, if you need more than one (for example, +to satisfy unique reporting requirements) be sure to list the primary +connection first as that is what Zuul will use to look up all changes +for that server. + +.. _drivers: + +Drivers +------- + +Drivers may support any of the following functions: + +* Sources -- hosts git repositories for projects. Zuul can clone git + repos for projects and fetch refs. +* Triggers -- emits events to which Zuul may respond. Triggers are + configured in pipelines to cause changes or other refs to be + enqueued. +* Reporters -- outputs information when a pipeline is finished + processing an item. + +Zuul includes the following drivers: + +.. toctree:: + :maxdepth: 2 + + drivers/gerrit + drivers/github + drivers/git + drivers/smtp + drivers/sql + drivers/timer + drivers/zuul diff --git a/doc/source/admin/drivers/gerrit.rst b/doc/source/admin/drivers/gerrit.rst new file mode 100644 index 0000000000..935cb32901 --- /dev/null +++ b/doc/source/admin/drivers/gerrit.rst @@ -0,0 +1,299 @@ +:title: Gerrit Driver + +Gerrit +====== + +`Gerrit`_ is a code review system. The Gerrit driver supports +sources, triggers, and reporters. + +.. _Gerrit: https://www.gerritcodereview.com/ + +Zuul will need access to a Gerrit user. + +Create an SSH keypair for Zuul to use if there isn't one already, and +create a Gerrit user with that key:: + + cat ~/id_rsa.pub | ssh -p29418 review.example.com gerrit create-account --ssh-key - --full-name Zuul zuul + +Give that user whatever permissions will be needed on the projects you +want Zuul to report on. For instance, you may want to grant +``Verified +/-1`` and ``Submit`` to the user. Additional categories +or values may be added to Gerrit. Zuul is very flexible and can take +advantage of those. + +Connection Configuration +------------------------ + +The supported options in ``zuul.conf`` connections are: + +.. attr:: + + .. attr:: driver + :required: + + .. value:: gerrit + + The connection must set ``driver=gerrit`` for Gerrit connections. + + .. attr:: server + + Fully qualified domain name of Gerrit server. + + .. attr:: canonical_hostname + + The canonical hostname associated with the git repos on the + Gerrit server. Defaults to the value of + :attr:`.server`. This is used to identify + projects from this connection by name and in preparing repos on + the filesystem for use by jobs. Note that Zuul will still only + communicate with the Gerrit server identified by ``server``; + this option is useful if users customarily use a different + hostname to clone or pull git repos so that when Zuul places + them in the job's working directory, they appear under this + directory name. + + .. attr:: port + :default: 29418 + + Gerrit server port. + + .. attr:: baseurl + + Path to Gerrit web interface. + + .. attr:: gitweb_url_template + :default: {baseurl}/gitweb?p={project.name}.git;a=commitdiff;h={sha} + + Url template for links to specific git shas. By default this will + point at Gerrit's built in gitweb but you can customize this value + to point elsewhere (like cgit or github). + + The three values available for string interpolation are baseurl + which points back to Gerrit, project and all of its safe attributes, + and sha which is the git sha1. + + .. attr:: user + :default: zuul + + User name to use when logging into Gerrit via ssh. + + .. attr:: sshkey + :default: ~zuul/.ssh/id_rsa + + Path to SSH key to use when logging into Gerrit. + + .. attr:: keepalive + :default: 60 + + SSH connection keepalive timeout; ``0`` disables. + +Trigger Configuration +--------------------- + +Zuul works with standard versions of Gerrit by invoking the ``gerrit +stream-events`` command over an SSH connection. It also reports back +to Gerrit using SSH. + +If using Gerrit 2.7 or later, make sure the user is a member of a group +that is granted the ``Stream Events`` permission, otherwise it will not +be able to invoke the ``gerrit stream-events`` command over SSH. + +.. attr:: pipeline.trigger. + + The dictionary passed to the Gerrit pipeline ``trigger`` attribute + supports the following attributes: + + .. attr:: event + :required: + + The event name from gerrit. Examples: ``patchset-created``, + ``comment-added``, ``ref-updated``. This field is treated as a + regular expression. + + .. attr:: branch + + The branch associated with the event. Example: ``master``. + This field is treated as a regular expression, and multiple + branches may be listed. + + .. attr:: ref + + On ref-updated events, the branch parameter is not used, instead + the ref is provided. Currently Gerrit has the somewhat + idiosyncratic behavior of specifying bare refs for branch names + (e.g., ``master``), but full ref names for other kinds of refs + (e.g., ``refs/tags/foo``). Zuul matches this value exactly + against what Gerrit provides. This field is treated as a + regular expression, and multiple refs may be listed. + + .. attr:: ignore-deletes + :default: true + + When a branch is deleted, a ref-updated event is emitted with a + newrev of all zeros specified. The ``ignore-deletes`` field is a + boolean value that describes whether or not these newrevs + trigger ref-updated events. + + .. attr:: approval + + This is only used for ``comment-added`` events. It only matches + if the event has a matching approval associated with it. + Example: ``Code-Review: 2`` matches a ``+2`` vote on the code + review category. Multiple approvals may be listed. + + .. attr:: email + + This is used for any event. It takes a regex applied on the + performer email, i.e. Gerrit account email address. If you want + to specify several email filters, you must use a YAML list. + Make sure to use non greedy matchers and to escapes dots! + Example: ``email: ^.*?@example\.org$``. + + .. attr:: username + + This is used for any event. It takes a regex applied on the + performer username, i.e. Gerrit account name. If you want to + specify several username filters, you must use a YAML list. + Make sure to use non greedy matchers and to escapes dots. + Example: ``username: ^zuul$``. + + .. attr:: comment + + This is only used for ``comment-added`` events. It accepts a + list of regexes that are searched for in the comment string. If + any of these regexes matches a portion of the comment string the + trigger is matched. ``comment: retrigger`` will match when + comments containing ``retrigger`` somewhere in the comment text + are added to a change. + + .. attr:: require-approval + + This may be used for any event. It requires that a certain kind + of approval be present for the current patchset of the change + (the approval could be added by the event in question). It + follows the same syntax as :attr:`pipeline.require..approval`. For each specified criteria there must exist + a matching approval. + + .. attr:: reject-approval + + This takes a list of approvals in the same format as + :attr:`pipeline.trigger..require-approval` but + will fail to enter the pipeline if there is a matching approval. + +Reporter Configuration +---------------------- + +Zuul works with standard versions of Gerrit by invoking the +``gerrit`` command over an SSH connection. It reports back to +Gerrit using SSH. + +The dictionary passed to the Gerrit reporter is used for ``gerrit +review`` arguments, with the boolean value of ``true`` simply +indicating that the argument should be present without following it +with a value. For example, ``verified: 1`` becomes ``gerrit review +--verified 1`` and ``submit: true`` becomes ``gerrit review +--submit``. + +A :ref:`connection` that uses the gerrit driver must be +supplied to the trigger. + +Requirements Configuration +-------------------------- + +As described in :attr:`pipeline.require` and :attr:`pipeline.reject`, +pipelines may specify that items meet certain conditions in order to +be enqueued into the pipeline. These conditions vary according to the +source of the project in question. To supply requirements for changes +from a Gerrit source named ``my-gerrit``, create a configuration such +as the following: + +.. code-block:: yaml + + pipeline: + require: + my-gerrit: + approval: + - Code-Review: 2 + +This indicates that changes originating from the Gerrit connection +named ``my-gerrit`` must have a ``Code-Review`` vote of ``+2`` in +order to be enqueued into the pipeline. + +.. attr:: pipeline.require. + + The dictionary passed to the Gerrit pipeline `require` attribute + supports the following attributes: + + .. attr:: approval + + This requires that a certain kind of approval be present for the + current patchset of the change (the approval could be added by + the event in question). It takes several sub-parameters, all of + which are optional and are combined together so that there must + be an approval matching all specified requirements. + + .. attr:: username + + If present, an approval from this username is required. It is + treated as a regular expression. + + .. attr:: email + + If present, an approval with this email address is required. It is + treated as a regular expression. + + .. attr:: older-than + + If present, the approval must be older than this amount of time + to match. Provide a time interval as a number with a suffix of + "w" (weeks), "d" (days), "h" (hours), "m" (minutes), "s" + (seconds). Example ``48h`` or ``2d``. + + .. attr:: newer-than + + If present, the approval must be newer than this amount + of time to match. Same format as "older-than". + + Any other field is interpreted as a review category and value + pair. For example ``Verified: 1`` would require that the + approval be for a +1 vote in the "Verified" column. The value + may either be a single value or a list: ``Verified: [1, 2]`` + would match either a +1 or +2 vote. + + .. attr:: open + + A boolean value (``true`` or ``false``) that indicates whether + the change must be open or closed in order to be enqueued. + + .. attr:: current-patchset + + A boolean value (``true`` or ``false``) that indicates whether the + change must be the current patchset in order to be enqueued. + + .. attr:: status + + A string value that corresponds with the status of the change + reported by the trigger. + +.. attr:: pipeline.reject. + + The `reject` attribute is the mirror of the `require` attribute. It + also accepts a dictionary under the connection name. This + dictionary supports the following attributes: + + .. attr:: approval + + This takes a list of approvals. If an approval matches the + provided criteria the change can not be entered into the + pipeline. It follows the same syntax as + :attr:`pipeline.require..approval`. + + Example to reject a change with any negative vote: + + .. code-block:: yaml + + reject: + my-gerrit: + approval: + - Code-Review: [-1, -2] diff --git a/doc/source/admin/drivers/git.rst b/doc/source/admin/drivers/git.rst new file mode 100644 index 0000000000..e0acec1168 --- /dev/null +++ b/doc/source/admin/drivers/git.rst @@ -0,0 +1,59 @@ +:title: Git Driver + +Git +=== + +This driver can be used to load Zuul configuration from public Git repositories, +for instance from ``openstack-infra/zuul-jobs`` that is suitable for use by +any Zuul system. It can also be used to trigger jobs from ``ref-updated`` events +in a pipeline. + +Connection Configuration +------------------------ + +The supported options in ``zuul.conf`` connections are: + +.. attr:: + + .. attr:: driver + :required: + + .. value:: git + + The connection must set ``driver=git`` for Git connections. + + .. attr:: baseurl + + Path to the base Git URL. Git repos name will be appended to it. + + .. attr:: poll_delay + :default: 7200 + + The delay in seconds of the Git repositories polling loop. + +Trigger Configuration +--------------------- + +.. attr:: pipeline.trigger. + + The dictionary passed to the Git pipeline ``trigger`` attribute + supports the following attributes: + + .. attr:: event + :required: + + Only ``ref-updated`` is supported. + + .. attr:: ref + + On ref-updated events, a ref such as ``refs/heads/master`` or + ``^refs/tags/.*$``. This field is treated as a regular expression, + and multiple refs may be listed. + + .. attr:: ignore-deletes + :default: true + + When a ref is deleted, a ref-updated event is emitted with a + newrev of all zeros specified. The ``ignore-deletes`` field is a + boolean value that describes whether or not these newrevs + trigger ref-updated events. diff --git a/doc/source/admin/drivers/github.rst b/doc/source/admin/drivers/github.rst new file mode 100644 index 0000000000..4f46af6941 --- /dev/null +++ b/doc/source/admin/drivers/github.rst @@ -0,0 +1,458 @@ +:title: GitHub Driver + +GitHub +====== + +The GitHub driver supports sources, triggers, and reporters. It can +interact with the public GitHub service as well as site-local +installations of GitHub enterprise. + +Configure GitHub +---------------- + +There are two options currently available. GitHub's project owner can either +manually setup web-hook or install a GitHub Application. In the first case, +the project's owner needs to know the zuul endpoint and the webhook secrets. + + +Web-Hook +........ + +To configure a project's `webhook events +`_: + +* Set *Payload URL* to + ``http:///connection//payload``. + +* Set *Content Type* to ``application/json``. + +Select *Events* you are interested in. See below for the supported events. + +You will also need to have a GitHub user created for your zuul: + +* Zuul public key needs to be added to the GitHub account + +* A api_token needs to be created too, see this `article + `_ + +Then in the zuul.conf, set webhook_token and api_token. + +Application +........... + +To create a `GitHub application +`_: + +* Go to your organization settings page to create the application, e.g.: + https://github.com/organizations/my-org/settings/apps/new + +* Set GitHub App name to "my-org-zuul" + +* Set Setup URL to your setup documentation, when user install the application + they are redirected to this url + +* Set Webhook URL to + ``http:///connection//payload``. + +* Create a Webhook secret + +* Set permissions: + + * Commit statuses: Read & Write + + * Issues: Read & Write + + * Pull requests: Read & Write + + * Repository contents: Read & Write (write to let zuul merge change) + +* Set events subscription: + + * Label + + * Status + + * Issue comment + + * Issues + + * Pull request + + * Pull request review + + * Pull request review comment + + * Commit comment + + * Create + + * Push + + * Release + +* Set Where can this GitHub App be installed to "Any account" + +* Create the App + +* Generate a Private key in the app settings page + +Then in the zuul.conf, set webhook_token, app_id and app_key. +After restarting zuul-scheduler, verify in the 'Advanced' tab that the +Ping payload works (green tick and 200 response) + +Users can now install the application using its public page, e.g.: +https://github.com/apps/my-org-zuul + + +Connection Configuration +------------------------ + +There are two forms of operation. Either the Zuul installation can be +configured as a `Github App`_ or it can be configured as a Webhook. + +If the `Github App`_ approach is taken, the config settings ``app_id`` and +``app_key`` are required. If the Webhook approach is taken, the ``api_token`` +setting is required. + +The supported options in ``zuul.conf`` connections are: + +.. attr:: + + .. attr:: driver + :required: + + .. value:: github + + The connection must set ``driver=github`` for GitHub connections. + + .. attr:: app_id + + App ID if you are using a *GitHub App*. Can be found under the + **Public Link** on the right hand side labeled **ID**. + + .. attr:: app_key + + Path to a file containing the secret key Zuul will use to create + tokens for the API interactions. In Github this is known as + **Private key** and must be collected when generated. + + .. attr:: api_token + + API token for accessing GitHub if Zuul is configured with + Webhooks. See `Creating an access token for command-line use + `_. + + .. attr:: webhook_token + + Required token for validating the webhook event payloads. In + the GitHub App Configuration page, this is called **Webhook + secret**. See `Securing your webhooks + `_. + + .. attr:: sshkey + :default: ~/.ssh/id_rsa + + Path to SSH key to use when cloning github repositories. + + .. attr:: server + :default: github.com + + Hostname of the github install (such as a GitHub Enterprise). + + .. attr:: canonical_hostname + + The canonical hostname associated with the git repos on the + GitHub server. Defaults to the value of :attr:`.server`. This is used to identify projects from + this connection by name and in preparing repos on the filesystem + for use by jobs. Note that Zuul will still only communicate + with the GitHub server identified by **server**; this option is + useful if users customarily use a different hostname to clone or + pull git repos so that when Zuul places them in the job's + working directory, they appear under this directory name. + + .. attr:: verify_ssl + :default: true + + Enable or disable ssl verification for GitHub Enterprise. This + is useful for a connection to a test installation. + +Trigger Configuration +--------------------- +GitHub webhook events can be configured as triggers. + +A connection name with the GitHub driver can take multiple events with +the following options. + +.. attr:: pipeline.trigger. + + The dictionary passed to the GitHub pipeline ``trigger`` attribute + supports the following attributes: + + .. attr:: event + :required: + + The event from github. Supported events are: + + .. value:: pull_request + + .. value:: pull_request_review + + .. value:: push + + .. attr:: action + + A :value:`pipeline.trigger..event.pull_request` + event will have associated action(s) to trigger from. The + supported actions are: + + .. value:: opened + + Pull request opened. + + .. value:: changed + + Pull request synchronized. + + .. value:: closed + + Pull request closed. + + .. value:: reopened + + Pull request reopened. + + .. value:: comment + + Comment added to pull request. + + .. value:: labeled + + Label added to pull request. + + .. value:: unlabeled + + Label removed from pull request. + + .. value:: status + + Status set on commit. + + A :value:`pipeline.trigger..event.pull_request_review` event will have associated + action(s) to trigger from. The supported actions are: + + .. value:: submitted + + Pull request review added. + + .. value:: dismissed + + Pull request review removed. + + .. attr:: branch + + The branch associated with the event. Example: ``master``. This + field is treated as a regular expression, and multiple branches + may be listed. Used for ``pull_request`` and + ``pull_request_review`` events. + + .. attr:: comment + + This is only used for ``pull_request`` ``comment`` actions. It + accepts a list of regexes that are searched for in the comment + string. If any of these regexes matches a portion of the comment + string the trigger is matched. ``comment: retrigger`` will + match when comments containing 'retrigger' somewhere in the + comment text are added to a pull request. + + .. attr:: label + + This is only used for ``labeled`` and ``unlabeled`` + ``pull_request`` actions. It accepts a list of strings each of + which matches the label name in the event literally. ``label: + recheck`` will match a ``labeled`` action when pull request is + labeled with a ``recheck`` label. ``label: 'do not test'`` will + match a ``unlabeled`` action when a label with name ``do not + test`` is removed from the pull request. + + .. attr:: state + + This is only used for ``pull_request_review`` events. It + accepts a list of strings each of which is matched to the review + state, which can be one of ``approved``, ``comment``, or + ``request_changes``. + + .. attr:: status + + This is used for ``pull-request`` and ``status`` actions. It + accepts a list of strings each of which matches the user setting + the status, the status context, and the status itself in the + format of ``user:context:status``. For example, + ``zuul_github_ci_bot:check_pipeline:success``. + + .. attr:: ref + + This is only used for ``push`` events. This field is treated as + a regular expression and multiple refs may be listed. GitHub + always sends full ref name, eg. ``refs/tags/bar`` and this + string is matched against the regular expression. + +Reporter Configuration +---------------------- +Zuul reports back to GitHub via GitHub API. Available reports include a PR +comment containing the build results, a commit status on start, success and +failure, an issue label addition/removal on the PR, and a merge of the PR +itself. Status name, description, and context is taken from the pipeline. + +.. attr:: pipeline.. + + To report to GitHub, the dictionaries passed to any of the pipeline + :ref:`reporter` attributes support the following + attributes: + + .. attr:: status + + String value (``pending``, ``success``, ``failure``) that the + reporter should set as the commit status on github. + + .. TODO support role markup in :default: so we can xref + :attr:`webapp.status_url` below + + .. attr:: status-url + :default: webapp.status_url or the empty string + + String value for a link url to set in the github + status. Defaults to the zuul server status_url, or the empty + string if that is unset. + + .. attr:: comment + :default: true + + Boolean value that determines if the reporter should add a + comment to the pipeline status to the github pull request. Only + used for Pull Request based items. + + .. attr:: merge + :default: false + + Boolean value that determines if the reporter should merge the + pull reqeust. Only used for Pull Request based items. + + .. attr:: label + + List of strings each representing an exact label name which + should be added to the pull request by reporter. Only used for + Pull Request based items. + + .. attr:: unlabel + + List of strings each representing an exact label name which + should be removed from the pull request by reporter. Only used + for Pull Request based items. + +.. _Github App: https://developer.github.com/apps/ + +Requirements Configuration +-------------------------- + +As described in :attr:`pipeline.require` and :attr:`pipeline.reject`, +pipelines may specify that items meet certain conditions in order to +be enqueued into the pipeline. These conditions vary according to the +source of the project in question. To supply requirements for changes +from a GitHub source named ``my-github``, create a congfiguration such +as the following:: + + pipeline: + require: + my-github: + review: + - type: approval + +This indicates that changes originating from the GitHub connection +named ``my-github`` must have an approved code review in order to be +enqueued into the pipeline. + +.. attr:: pipeline.require. + + The dictionary passed to the GitHub pipeline `require` attribute + supports the following attributes: + + .. attr:: review + + This requires that a certain kind of code review be present for + the pull request (it could be added by the event in question). + It takes several sub-parameters, all of which are optional and + are combined together so that there must be a code review + matching all specified requirements. + + .. attr:: username + + If present, a code review from this username is required. It + is treated as a regular expression. + + .. attr:: email + + If present, a code review with this email address is + required. It is treated as a regular expression. + + .. attr:: older-than + + If present, the code review must be older than this amount of + time to match. Provide a time interval as a number with a + suffix of "w" (weeks), "d" (days), "h" (hours), "m" + (minutes), "s" (seconds). Example ``48h`` or ``2d``. + + .. attr:: newer-than + + If present, the code review must be newer than this amount of + time to match. Same format as "older-than". + + .. attr:: type + + If present, the code review must match this type (or types). + + .. TODO: what types are valid? + + .. attr:: permission + + If present, the author of the code review must have this + permission (or permissions). The available values are + ``read``, ``write``, and ``admin``. + + .. attr:: open + + A boolean value (``true`` or ``false``) that indicates whether + the change must be open or closed in order to be enqueued. + + .. attr:: current-patchset + + A boolean value (``true`` or ``false``) that indicates whether + the item must be associated with the latest commit in the pull + request in order to be enqueued. + + .. TODO: this could probably be expanded upon -- under what + circumstances might this happen with github + + .. attr:: status + + A string value that corresponds with the status of the pull + request. The syntax is ``user:status:value``. + + .. attr:: label + + A string value indicating that the pull request must have the + indicated label (or labels). + +.. attr:: pipeline.reject. + + The `reject` attribute is the mirror of the `require` attribute. It + also accepts a dictionary under the connection name. This + dictionary supports the following attributes: + + .. attr:: review + + This takes a list of code reviews. If a code review matches the + provided criteria the pull request can not be entered into the + pipeline. It follows the same syntax as + :attr:`pipeline.require..review` diff --git a/doc/source/admin/drivers/smtp.rst b/doc/source/admin/drivers/smtp.rst new file mode 100644 index 0000000000..11c062473c --- /dev/null +++ b/doc/source/admin/drivers/smtp.rst @@ -0,0 +1,87 @@ +:title: SMTP Driver + +SMTP +==== + +The SMTP driver supports reporters only. It is used to send email +when items report. + +Connection Configuration +------------------------ + +.. attr:: + + .. attr:: driver + :required: + + .. value:: smtp + + The connection must set ``driver=smtp`` for SMTP connections. + + .. attr:: server + :default: localhost + + SMTP server hostname or address to use. + + .. attr:: port + :default: 25 + + SMTP server port. + + .. attr:: default_from + :default: zuul + + Who the email should appear to be sent from when emailing the report. + This can be overridden by individual pipelines. + + .. attr:: default_to + :default: zuul + + Who the report should be emailed to by default. + This can be overridden by individual pipelines. + +Reporter Configuration +---------------------- + +A simple email reporter is also available. + +A :ref:`connection` that uses the smtp driver must be supplied to the +reporter. The connection also may specify a default *To* or *From* +address. + +Each pipeline can overwrite the ``subject`` or the ``to`` or ``from`` address by +providing alternatives as arguments to the reporter. For example: + +.. code-block:: yaml + + - pipeline: + name: post-merge + success: + outgoing_smtp: + to: you@example.com + failure: + internal_smtp: + to: you@example.com + from: alternative@example.com + subject: Change {change} failed + +.. attr:: pipeline.. + + To report via email, the dictionaries passed to any of the pipeline + :ref:`reporter` attributes support the following + attributes: + + .. attr:: to + + The SMTP recipient address for the report. Multiple addresses + may be specified as one value separated by commas. + + .. attr:: from + + The SMTP sender address for the report. + + .. attr:: subject + + The Subject of the report email. + + .. TODO: document subject string formatting. diff --git a/doc/source/admin/drivers/sql.rst b/doc/source/admin/drivers/sql.rst new file mode 100644 index 0000000000..b9ce24bc9b --- /dev/null +++ b/doc/source/admin/drivers/sql.rst @@ -0,0 +1,85 @@ +:title: SQL Driver + +SQL +=== + +The SQL driver supports reporters only. Only one connection per +database is permitted. + +Connection Configuration +------------------------ + +The connection options for the SQL driver are: + +.. attr:: + + .. attr:: driver + :required: + + .. value:: sql + + The connection must set ``driver=sql`` for SQL connections. + + .. attr:: dburi + :required: + + Database connection information in the form of a URI understood + by SQLAlchemy. See `The SQLAlchemy manual + `_ + for more information. + + The driver will automatically set up the database creating and managing + the necesssary tables. Therefore the provided user should have sufficient + permissions to manage the database. For example: + + .. code-block:: sql + + GRANT ALL ON my_database TO 'my_user'@'%'; + + .. attr:: pool_recycle + :default: 1 + + Tune the pool_recycle value. See `The SQLAlchemy manual on pooling + `_ + for more information. + + .. attr:: table_prefix + :default: '' + + The string to prefix the table names. This makes it possible to run + several zuul deployments against the same database. This can be useful + if you rely on external databases which you don't have under control. + The default is to have no prefix. + +Reporter Configuration +---------------------- + +This reporter is used to store results in a database. + +A :ref:`connection` that uses the sql driver must be +supplied to the reporter. + +``zuul.conf`` contains the database connection and credentials. To +store different reports in different databases you'll need to create a +new connection per database. + +The SQL reporter does nothing on :attr:`pipeline.start` or +:attr:`pipeline.merge-failure`; it only acts on +:attr:`pipeline.success` or :attr:`pipeline.failure` reporting stages. + +For example: + +.. code-block:: yaml + + - pipeline: + name: post-merge + success: + mydb_conn: + failure: + mydb_conn: + +.. attr:: pipeline.. + + To report to a database, add a key with the connection name and an + empty value to the desired pipeline :ref:`reporter` + attributes. diff --git a/doc/source/admin/drivers/timer.rst b/doc/source/admin/drivers/timer.rst new file mode 100644 index 0000000000..3b38c99a0c --- /dev/null +++ b/doc/source/admin/drivers/timer.rst @@ -0,0 +1,30 @@ +:title: Timer Driver + +Timer +===== + +The timer driver supports triggers only. It is used for configuring +pipelines so that jobs run at scheduled times. No connection +configuration is required. + +Trgger Configuration +-------------------- + +Timers don't require a special connection or driver. Instead they can +simply be used by listing ``timer`` as the trigger. + +This trigger will run based on a cron-style time specification. It +will enqueue an event into its pipeline for every project defined in +the configuration. Any job associated with the pipeline will run in +response to that event. + +.. attr:: pipeline.trigger.timer + + The timer trigger supports the following attributes: + + .. attr:: time + :required: + + The time specification in cron syntax. Only the 5 part syntax + is supported, not the symbolic names. Example: ``0 0 * * *`` + runs at midnight. The first weekday is Monday. diff --git a/doc/source/admin/drivers/zuul.rst b/doc/source/admin/drivers/zuul.rst new file mode 100644 index 0000000000..41535ee063 --- /dev/null +++ b/doc/source/admin/drivers/zuul.rst @@ -0,0 +1,44 @@ +:title: Zuul Driver + +Zuul +==== + +The Zuul driver supports triggers only. It is used for triggering +pipelines based on internal Zuul events. + +Trigger Configuration +--------------------- + +Zuul events don't require a special connection or driver. Instead they +can simply be used by listing ``zuul`` as the trigger. + +.. attr:: pipeline.trigger.zuul + + The Zuul trigger supports the following attributes: + + .. attr:: event + :required: + + The event name. Currently supported events: + + .. value:: project-change-merged + + When Zuul merges a change to a project, it generates this + event for every open change in the project. + + .. warning:: + + Triggering on this event can cause poor performance when + using the GitHub driver with a large number of + installations. + + .. value:: parent-change-enqueued + + When Zuul enqueues a change into any pipeline, it generates + this event for every child of that change. + + .. attr:: pipeline + + Only available for ``parent-change-enqueued`` events. This is + the name of the pipeline in which the parent change was + enqueued. diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst new file mode 100644 index 0000000000..a2a2ee7720 --- /dev/null +++ b/doc/source/admin/index.rst @@ -0,0 +1,20 @@ +Administrator's Guide +===================== + +This guide is intended for administrators of Zuul systems. It covers +installation, operation, and the portion of Zuul configuration that +occurs outside of the projects upon which Zuul operates. Advanced +users may be interested in some of the concepts described here, as +well as understanding what features the underlying configuration +provides to in-project configuration. + +.. toctree:: + :maxdepth: 2 + + quick-start + installation + components + connections + tenants + monitoring + client diff --git a/doc/source/admin/installation.rst b/doc/source/admin/installation.rst new file mode 100644 index 0000000000..ae7d571de5 --- /dev/null +++ b/doc/source/admin/installation.rst @@ -0,0 +1,69 @@ +Installation +============ + +Install Zuul +------------ + +To install a Zuul release from PyPI, run:: + + pip install zuul + +Or from a git checkout, run:: + + pip install . + +That will also install Zuul's python dependencies. To minimize +interaction with other python packages installed on a system, you may +wish to install Zuul within a Python virtualenv. + +Zuul has several system-level dependencies as well. You can find a +list of operating system packages in `bindep.txt` in Zuul's source +directory. + +External Dependencies +--------------------- + +Zuul interacts with several other systems described below. + +Gearman +~~~~~~~ + +Gearman is a job distribution system that Zuul uses to communicate +with its distributed components. The Zuul scheduler distributes work +to Zuul mergers and executors using Gearman. You may supply your own +gearman server, but the Zuul scheduler includes a built-in server +which is recommended. Ensure that all Zuul hosts can communicate with +the gearman server. + +Zuul distributes secrets to executors via gearman, so be sure to +secure it with TLS and certificate authentication. Obtain (or +generate) a certificate for both the server and the clients (they may +use the same certificate or have individual certificates). They must +be signed by a CA, but it can be your own CA. + +Nodepool +~~~~~~~~ + +In order to run all but the simplest jobs, Zuul uses a companion +program, Nodepool, to supply the nodes (whether dynamic cloud +instances or static hardware) used by jobs. Before starting Zuul, +ensure you have Nodepool installed and any images you require built. +Zuul only makes one requirement of these nodes: that it be able to log +in given a username and ssh private key. + +.. TODO: SpamapS any zookeeper config recommendations? + +Nodepool uses Zookeeper to communicate internally among its +components, and also to communicate with Zuul. You can run a simple +single-node Zookeeper instance, or a multi-node cluster. Ensure that +the host running the Zuul scheduler has access to the cluster. + +Ansible +~~~~~~~ + +Zuul uses Ansible to run jobs. Each version of Zuul is designed to +work with a specific, contemporary version of Ansible. Zuul specifies +that version of Ansible in its python package metadata, and normally +the correct version will be installed automatically with Zuul. +Because of the close integration of Zuul and Ansible, attempting to +use other versions of Ansible with Zuul is not recommended. diff --git a/doc/source/admin/monitoring.rst b/doc/source/admin/monitoring.rst new file mode 100644 index 0000000000..e6e6139878 --- /dev/null +++ b/doc/source/admin/monitoring.rst @@ -0,0 +1,278 @@ +:title: Monitoring + +Monitoring +========== + +.. _statsd: + +Statsd reporting +---------------- + +Zuul comes with support for the statsd protocol, when enabled and configured +(see below), the Zuul scheduler will emit raw metrics to a statsd receiver +which let you in turn generate nice graphics. + +Configuration +~~~~~~~~~~~~~ + +Statsd support uses the ``statsd`` python module. Note that support +is optional and Zuul will start without the statsd python module +present. + +Configuration is in the :attr:`statsd` section of ``zuul.conf``. + +Metrics +~~~~~~~ + +These metrics are emitted by the Zuul :ref:`scheduler`: + +.. stat:: zuul.event..event. + :type: counter + + Zuul will report counters for each type of event it receives from + each of its configured drivers. + +.. stat:: zuul.tenant..pipeline + + Holds metrics specific to jobs. This hierarchy includes: + + .. stat:: + + A set of metrics for each pipeline named as defined in the Zuul + config. + + .. stat:: all_jobs + :type: counter + + Number of jobs triggered by the pipeline. + + .. stat:: current_changes + :type: gauge + + The number of items currently being processed by this + pipeline. + + .. stat:: project + + This hierarchy holds more specific metrics for each project + participating in the pipeline. + + .. stat:: + + The canonical hostname for the triggering project. + Embedded ``.`` characters will be translated to ``_``. + + .. stat:: + + The name of the triggering project. Embedded ``/`` or + ``.`` characters will be translated to ``_``. + + .. stat:: + + The name of the triggering branch. Embedded ``/`` or + ``.`` characters will be translated to ``_``. + + .. stat:: job + + Subtree detailing per-project job statistics: + + .. stat:: + + The triggered job name. + + .. stat:: + :type: counter, timer + + A counter for each type of result (e.g., ``SUCCESS`` or + ``FAILURE``, ``ERROR``, etc.) for the job. If the + result is ``SUCCESS`` or ``FAILURE``, Zuul will + additionally report the duration of the build as a + timer. + + .. stat:: current_changes + :type: gauge + + The number of items of this project currently being + processed by this pipeline. + + .. stat:: resident_time + :type: timer + + A timer metric reporting how long each item for this + project has been in the pipeline. + + .. stat:: total_changes + :type: counter + + The number of changes for this project processed by the + pipeline since Zuul started. + + .. stat:: resident_time + :type: timer + + A timer metric reporting how long each item has been in the + pipeline. + + .. stat:: total_changes + :type: counter + + The number of changes processed by the pipeline since Zuul + started. + + .. stat:: wait_time + :type: timer + + How long each item spent in the pipeline before its first job + started. + +.. stat:: zuul.executor. + + Holds metrics emitted by individual executors. The ```` + component of the key will be replaced with the hostname of the + executor. + + .. stat:: builds + :type: counter + + Incremented each time the executor starts a build. + + .. stat:: running_builds + :type: gauge + + The number of builds currently running on this executor. + + .. stat:: load_average + :type: gauge + + The one-minute load average of this executor, multiplied by 100. + +.. stat:: zuul.nodepool + + Holds metrics related to Zuul requests from Nodepool. + + .. stat:: requested + :type: counter + + Incremented each time a node request is submitted to Nodepool. + + .. stat:: label.