system-config/zuul.d/system-config-run.yaml
Monty Taylor d93a661ae4 Run iptables in service playbooks instead of base
It's the only part of base that's important to run when we run a
service. Run it in the service playbooks and get rid of the
dependency on infra-prod-base.

Continue running it in base so that new nodes are brought up
with iptables in place.

Bump the timeout for the mirror job, because the iptables addition
seems to have just bumped it over the edge.

Change-Id: I4608216f7a59cfa96d3bdb191edd9bc7bb9cca39
2020-06-04 07:44:22 -05:00

757 lines
23 KiB
YAML

- job:
name: system-config-run
description: |
Run the "base" playbook for system-config hosts.
This is a parent job designed to be inherited.
abstract: true
pre-run: playbooks/zuul/run-base-pre.yaml
run: playbooks/zuul/run-base.yaml
post-run: playbooks/zuul/run-base-post.yaml
vars:
install_ansible_ara_enable: true
zuul_copy_output: "{{ copy_output | combine(host_copy_output | default({})) }}"
stage_dir: "{{ ansible_user_dir }}/zuul-output"
copy_output:
'/var/log/syslog': logs_txt
'/var/log/messages': logs_txt
'/var/log/docker': logs
'/etc/iptables/rules.v4': logs_txt
'/etc/iptables/rules.v6': logs_txt
host-vars:
bridge.openstack.org:
host_copy_output:
'{{ zuul.project.src_dir }}/junit.xml': logs
# Note: the following two jobs implement the variant-based multiple
# inheritance trick. Both of these variants will always apply,
# therefore both parents will appear in the inheritance hierarchy).
- job:
name: system-config-run-containers
parent: system-config-run
# Note: see above re multiple-inheritance.
- job:
name: system-config-run-containers
parent: opendev-buildset-registry-consumer
description: |
Run the "base" playbook for system-config hosts which use
containers.
This is a parent job designed to be inherited. Use this job if
the service in question is container-based. It expects a
buildset registry and pulls images from the intermediate
registry.
- job:
name: system-config-run-base
parent: system-config-run
description: |
Run the "base" playbook on each of the node types
currently in use.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: xenial
label: ubuntu-xenial
- name: bionic
label: ubuntu-bionic
- name: focal
label: ubuntu-focal
files:
- playbooks/
- roles/
- testinfra/
- job:
name: system-config-run-base-ansible-devel
parent: system-config-run-base
description: |
Run the base playbook with the latest ansible.
required-projects:
- name: github.com/ansible/ansible
override-checkout: devel
- name: github.com/philpep/testinfra
- name: openstack/openstacksdk
- name: recordsansible/ara
# NOTE(ianw): 2019-06-11 pinned to stable branch until we handle 1.0
# http://lists.openstack.org/pipermail/openstack-infra/2019-June/006400.html
override-checkout: stable/0.x
vars:
bridge_ansible_name: '{{ ansible_user_dir}}/src/github.com/ansible/ansible'
bridge_ansible_version: null
bridge_openstacksdk_name: '{{ ansible_user_dir }}/src/opendev.org/openstack/openstacksdk'
bridge_openstacksdk_version: null
bridge_ara_name: '{{ ansible_user_dir}}/src/opendev.org/recordsansible/ara'
bridge_ara_version: null
# Although we don't have an arm64 based bridge; Zuul can't currently
# allocate a mixed x86/arm64 situation across clouds. Thus it helps
# to keep this clean so we can run the other tests.
- job:
name: system-config-run-base-arm64
parent: system-config-run
description: |
Run the "base" playbook on ARM64.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic-arm64
- name: bionic
label: ubuntu-bionic-arm64
files:
- playbooks/
- roles/
- testinfra/
- job:
name: system-config-run-eavesdrop
parent: system-config-run-containers
description: |
Run the playbook for an eavesdrop server.
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
- openstack/project-config
requires: accessbot-container-image
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: eavesdrop01.openstack.org
label: ubuntu-xenial
vars:
run_playbooks:
- playbooks/service-eavesdrop.yaml
files:
- playbooks/service-eavesdrop.yaml
- playbooks/run-accessbot.yaml
- playbooks/group_vars/eavesdrop.yaml
- playbooks/group_vars/puppet.yaml
- playbooks/roles/zuul-user
- playbooks/roles/install-docker
- playbooks/roles/puppet-install/
- playbooks/roles/disable-puppet-agent/
- playbooks/roles/accessbot
- playbooks/roles/logrotate
- modules/openstack_project/manifests/eavesdrop.pp
- manifests/eavesdrop.pp
- docker/accessbot/
- testinfra/test_eavesdrop.py
- job:
name: system-config-run-codesearch
parent: system-config-run
description: |
Run the playbook for an codesearch server.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: codesearch01.openstack.org
label: ubuntu-xenial
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
- openstack/project-config
files:
- playbooks/install-ansible.yaml
- playbooks/service-codesearch.yaml
- playbooks/group_vars/puppet.yaml
- playbooks/roles/run-puppet/
- playbooks/roles/install-ansible-roles/
- playbooks/roles/sync-project-config
- playbooks/roles/puppet-install/
- playbooks/roles/disable-puppet-agent/
- modules/openstack_project/manifests/codesearch.pp
- modules/openstack_project/files/resync-hound-config.sh
- manifests/codesearch.pp
vars:
run_playbooks:
- playbooks/service-codesearch.yaml
- job:
name: system-config-run-letsencrypt
parent: system-config-run
description: |
Run the playbook for letsencrypt key acquisition
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns-letsencrypt.opendev.org
label: ubuntu-bionic
- name: letsencrypt01.opendev.org
label: ubuntu-bionic
- name: letsencrypt02.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-nameserver.yaml
- playbooks/service-letsencrypt.yaml
host-vars:
bridge.openstack.org:
host_copy_output:
'/var/lib/certcheck': logs
letsencrypt01.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
letsencrypt02.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/letsencrypt.yaml
- playbooks/roles/letsencrypt
- job:
name: system-config-run-lists
parent: system-config-run
description: |
Run the playbook for a list server.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: lists.openstack.org
label: ubuntu-xenial
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
files:
- playbooks/install-ansible.yaml
- modules/openstack_project/manifests/lists.pp
- playbooks/host_vars/lists.openstack.org.yaml
- playbooks/group_vars/puppet
- playbooks/roles/run-puppet/
- playbooks/roles/install-ansible-roles/
- playbooks/roles/base/exim
vars:
run_playbooks:
- playbooks/remote_puppet_else.yaml
- job:
name: system-config-run-nodepool
parent: system-config-run
description: |
Run the playbook for nodepool.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zk01.opendev.org
label: ubuntu-bionic
- name: nl01.openstack.org
label: ubuntu-xenial
- name: nb04.opendev.org
label: ubuntu-bionic
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
- openstack/project-config
host-vars:
nl01.openstack.org:
host_copy_output:
'/etc/nodepool/nodepool.yaml': logs
'/var/log/nodepool/launcher-debug.log': logs
nb04.opendev.org:
host_copy_output:
'/etc/nodepool/nodepool.yaml': logs
'/var/log/nodepool/builder-debug.log': logs
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-zookeeper.yaml
- playbooks/service-nodepool.yaml
- playbooks/remote_puppet_else.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/nodepool.yaml
- playbooks/group_vars/nodepool-builder.yaml
- playbooks/group_vars/nodepool-launcher.yaml
- playbooks/group_vars/puppet
- playbooks/roles/run-puppet/
- playbooks/roles/install-ansible-roles/
- playbooks/roles/configure-openstacksdk/
- playbooks/roles/nodepool
- playbooks/templates/clouds/
- testinfra/test_nodepool.py
- job:
name: system-config-run-dns
parent: system-config-run
description: |
Run the playbook for dns.
required-projects:
- opendev/zone-opendev.org
- opendev/zone-zuul-ci.org
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns1.opendev.org
label: ubuntu-bionic
- name: ns1.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-nameserver.yaml
host-vars:
adns1.opendev.org:
host_copy_output:
'/etc/bind/named.conf': logs
'/var/lib/bind/zones': logs
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/adns.yaml
- playbooks/group_vars/dns.yaml
- playbooks/host_vars/(ad)?ns\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/adns.yaml.j2
- playbooks/zuul/templates/group_vars/ns.yaml.j2
- playbooks/roles/master-nameserver/
- playbooks/roles/nameserver/
- testinfra/test_adns.py
- testinfra/test_ns.py
- job:
name: system-config-run-backup
parent: system-config-run
description: |
Run the playbook for backup configuration
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: backup01.region.provider.opendev.org
label: ubuntu-bionic
- name: backup-test01.opendev.org
label: ubuntu-bionic
- name: backup-test02.opendev.org
label: ubuntu-xenial
vars:
run_playbooks:
- playbooks/service-backup.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/roles/backup
- playbooks/zuul/templates/host_vars/backup
- testinfra/test_backups.py
- job:
name: system-config-run-mirror-base
parent: system-config-run
abstract: true
description: |
Run the playbook for a mirror node
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-mirror.yaml
host-vars:
mirror01.openafs.provider.opendev.org:
host_copy_output:
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
'/etc/apache2/sites-available/mirror.conf': logs
mirror02.openafs.provider.opendev.org:
host_copy_output:
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
'/etc/apache2/sites-available/mirror.conf': logs
files:
- playbooks/install-ansible.yaml
- roles/
- playbooks/group_vars/mirror.yaml
- playbooks/roles/mirror/
- playbooks/roles/letsencrypt
- playbooks/service-letsencrypt.yaml
- playbooks/service-mirror.yaml
- playbooks/zuul/templates/group_vars/mirror.yaml.j2
- testinfra/test_mirror.py
- job:
name: system-config-run-mirror-x86
parent: system-config-run-mirror-base
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: mirror01.openafs.provider.opendev.org
label: ubuntu-bionic
- name: mirror02.openafs.provider.opendev.org
label: ubuntu-focal
- job:
name: system-config-run-mirror-arm64
parent: system-config-run-mirror-base
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic-arm64
- name: mirror01.openafs.provider.opendev.org
label: ubuntu-bionic-arm64
- job:
name: system-config-run-mirror-update
parent: system-config-run
description: |
Run the playbook for a mirror update node
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: mirror-update01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-mirror-update.yaml
files:
- playbooks/install-ansible.yaml
- roles/
- playbooks/roles/mirror-update/
- playbooks/service-mirror-update.yaml
- testinfra/test_mirror-update.py
- job:
name: system-config-run-docker-registry
parent: system-config-run
description: |
Run the playbook for the docker registry.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: insecure-ci-registry01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-registry.yaml
host-vars:
insecure-ci-registry01.opendev.org:
host_copy_output:
'/var/registry/auth': logs
'/var/registry/certs': logs
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/registry.yaml
- playbooks/host_vars/insecure-ci-registry\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/registry.yaml.j2
- playbooks/roles/letsencrypt-create-certs/handlers/restart_zuul_registry.yaml
- playbooks/roles/registry/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- testinfra/test_registry.py
- job:
name: system-config-run-etherpad
parent: system-config-run-containers
description: |
Run the playbook for the etherpad servers.
timeout: 3600
requires: etherpad-container-image
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: etherpad01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-etherpad.yaml
files:
- playbooks/bridge.yaml
- playbooks/service-letsencrypt.yaml
- playbooks/service-etherpad.yaml
- playbooks/roles/etherpad/
- playbooks/roles/install-docker/
- playbooks/roles/pip3/
- docker/etherpad/
- job:
name: system-config-run-gitea
parent: system-config-run-containers
description: |
Run the playbook for the gitea servers.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: gitea-lb01.opendev.org
label: ubuntu-bionic
- name: gitea99.opendev.org
label: ubuntu-bionic
required-projects:
- openstack/project-config
- opendev/system-config
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-gitea-lb.yaml
- playbooks/service-gitea.yaml
- playbooks/manage-projects.yaml
run_test_playbook: playbooks/test-gitea.yaml
host-vars:
gitea99.opendev.org:
host_copy_output:
'/var/gitea/conf': logs
'/var/gitea/certs': logs
'/var/gitea/logs': logs
gitea-lb01.opendev.org:
host_copy_output:
'/var/haproxy/etc': logs
files:
- playbooks/install-ansible.yaml
- playbooks/service-letsencrypt.yaml
- playbooks/service-gitea-lb.yaml
- playbooks/service-gitea.yaml
- playbooks/manage-projects.yaml
- playbooks/test-gitea.yaml
- playbooks/group_vars/gitea.yaml
- playbooks/group_vars/gitea-lb.yaml
- playbooks/host_vars/gitea
- playbooks/zuul/templates/group_vars/gitea.yaml.j2
- playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- playbooks/roles/letsencrypt
- playbooks/roles/gerrit/
- playbooks/roles/gitea/
- playbooks/roles/gitea-git-repos/
- playbooks/roles/haproxy/
- playbooks/roles/letsencrypt-create-certs/handlers/restart_gitea.yaml
- testinfra/test_gitea.py
- testinfra/test_gitea_lb.py
# From gitea_files -- If we rebuild the image, we want to run
# this job as well.
- docker/gitea/
# From haproxy-statsd_files -- If we rebuild the image, we want
# to run this job as well.
- docker/haproxy-statsd/
- job:
name: system-config-run-meetpad
parent: system-config-run-containers
requires: jitsi-meet-container-image
description: |
Run the playbook for jitsi-meet.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: meetpad01.opendev.org
label: ubuntu-bionic
- name: jvb01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-meetpad.yaml
host-vars:
meetpad01.opendev.org:
host_copy_output:
'/var/jitsi-meet': logs
jvb01.opendev.org:
host_copy_output:
'/var/jitsi-meet': logs
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/meetpad.yaml
- playbooks/host_vars/meetpad\d+.opendev.org.yaml
- playbooks/roles/letsencrypt-create-certs/handlers/restart_jitsi_meet.yaml
- playbooks/roles/jitsi-meet/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- playbooks/zuul/templates/group_vars/meetpad.yaml.j2
- testinfra/test_meetpad.py
- docker/jitsi-meet/
- job:
name: system-config-run-zookeeper
parent: system-config-run
description: |
Run the playbook for the zookeeper cluster.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zk01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-zookeeper.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/zookeeper.yaml
- ^playbooks/host_vars/zk\d+\..*
- playbooks/roles/zookeeper/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- testinfra/test_zookeeper.py
- job:
name: system-config-run-zuul-preview
parent: system-config-run
description: |
Run the playbook for the zuul-preview service.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zp01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-zuul-preview.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/roles/zuul-preview/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- testinfra/test_zuul_preview.py
- job:
name: system-config-run-zuul
parent: system-config-run
description: |
Run the playbook for the main Zuul cluster.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zk01.opendev.org
label: ubuntu-bionic
- name: zm01.openstack.org
label: ubuntu-xenial
- name: ze01.opendev.org
label: ubuntu-focal
- name: ze01.openstack.org
label: ubuntu-xenial
- name: zuul01.openstack.org
label: ubuntu-xenial
required-projects:
- openstack/project-config
- opendev/system-config
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-zookeeper.yaml
- playbooks/service-zuul.yaml
host-vars:
zm01.openstack.org:
host_copy_output:
'/etc/zuul/zuul.conf': logs
'/var/log/zuul/merger-debug.log': logs
ze01.opendev.org:
host_copy_output:
'/etc/zuul/zuul.conf': logs
'/var/log/zuul/executor-debug.log': logs
ze01.openstack.org:
host_copy_output:
'/etc/zuul/zuul.conf': logs
'/var/log/zuul/executor-debug.log': logs
zuul01.openstack.org:
host_copy_output:
'/etc/zuul/zuul.conf': logs
'/var/log/zuul/debug.log': logs
files:
- playbooks/install-ansible.yaml
- playbooks/service-zookeeper.yaml
- playbooks/service-zuul.yaml
- playbooks/group_vars/zuul
- playbooks/group_vars/zookeeper.yaml
- playbooks/host_vars/zk\d+
- playbooks/host_vars/zuul01.openstack.org
- playbooks/roles/zookeeper/
- playbooks/roles/install-apt-repo
- playbooks/roles/zuul
- job:
name: system-config-run-review
parent: system-config-run-containers
description: |
Run the playbook for gerrit (in a container).
requires: gerrit-2.13-container-image
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: review01.openstack.org
label: ubuntu-xenial
- name: review-dev01.opendev.org
label: ubuntu-xenial
required-projects:
- openstack/project-config
- opendev/system-config
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-review-dev.yaml
- playbooks/service-review.yaml
host-vars:
review-dev01.opendev.org:
host_copy_output:
'/home/gerrit2/review_site/etc': logs
'/home/gerrit2/review_site/logs': logs
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
review01.openstack.org:
host_copy_output:
'/home/gerrit2/review_site/etc': logs
'/home/gerrit2/review_site/logs': logs
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
files:
- playbooks/install-ansible.yaml
- playbooks/service-letsencrypt.yaml
- playbooks/service-review.*.yaml
- playbooks/group_vars/review.yaml
- playbooks/host_vars/review\d+.openstack.org.yaml
- playbooks/group_vars/review-dev.yaml
- playbooks/host_vars/review-dev\d+.opendev.org.yaml
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- playbooks/roles/letsencrypt
- playbooks/roles/gerrit/
- playbooks/zuul/templates/group_vars/review-dev.yaml.j2
- playbooks/zuul/templates/group_vars/review.yaml.j2
- testinfra/test_gerrit.py
- docker/gerrit/2.13/
- job:
name: system-config-run-static
parent: system-config-run
description: |
Run the playbook for a static node.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: static01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-static.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/roles/static/
- playbooks/roles/letsencrypt
- playbooks/service-letsencrypt.yaml
- playbooks/service-static.yaml
- testinfra/test_static.py
host-vars:
static01.opendev.org:
host_copy_output:
'/var/log/acme.sh/': logs
'/etc/apache2/': logs
'/var/log/apache2/': logs