system-config/.zuul.yaml
Ian Wienand b85282c046 Move rsync mirror updates to new opendev.org mirror-update host
This move was prompted by wishing to expose the mirror update logs for
the rsync updates so that debugging problems does not require a root
user (note: not actually done in this change; will be a follow-on).

Rather than start hacking at puppet, the rsync mirror scripts make a
nice delination point for starting an Ansible-first/Bionic update.

Most magic is included in the scripts, so there is not much more to do
than copy them.  The host uses the existing kerberos and openafs roles
and copies the key material into place (to be added before merge).

Note the scripts are removed from the extant puppet so we don't have
two updates happening simultaneously.  This will also require a manual
clean to remove the cron jobs as a once-off when merging.

The other part of mirror-update is the reprepro based scripts for the
various debuntu repositories.  They are left as future work for now.

Testing is added to ensure dependencies and scripts are all in place.

Change-Id: I525ac18b55f0e11b0a541b51fa97ee5d6512bf70
2019-07-02 16:42:33 +10:00

788 lines
24 KiB
YAML

- job:
name: puppet-beaker-rspec-puppet-4-infra-system-config
parent: puppet-beaker-rspec-puppet-4-infra
vars:
project_src_dir: "{{ zuul.project.src_dir }}/modules/openstack_project"
# Image building jobs
- secret:
name: system-config-dockerhub
data:
username: opendevzuul
password: !encrypted/pkcs1-oaep
- FF2mwsdYo4b9QI/kEmeFdZS32I+OJUac2xy4JEQJaIevRK9E3rQ/11cBn7xX6zelA/htY
hENMM1gGTYO8BKgTVNijFfCiVCTYFJ9efsOrSeMdj2aLcq/OkZFeq8gMHAZDv+3qaiggq
hbDmldgSouNahhJByWMsDjYqnItfR8w1dYisNQBkDEJdNW5+Cin5WgHRYv2skui9pjumU
vWHjwLbITLEjb0hwVErggAzvED4x0UjNEMfb6KCpoXeKdlCaI8h1DXVD9RdQ4s1WvUCro
xwl4AIUyCmdtXbqW787PQ4j8BgdOi8RD40ZTSxpoV/bwT+/NrP3Jj2KVYdEKQJimzXnwL
goDclapu9aFrAaqwWSnWCRc7GQptSZ7DBXmA7MsNinBclWHX/chX5k9Q1+XifAhmKJZqP
ovkzEGtmUjTQZFjvuCtFxIWZEGVtGHDmgEkzevgVCeuHDTeIDlkAvA3+TT+/A70zyYdbP
EHOTSSjCU6RSlJxAXjNb5pzKjpeT7PyX2f4/8Zn3JnwTlfPJ0C7qdEYjQ54hbsKynjrpQ
7f6NN9pAeQS1pid5J7oH4f7YaKM78pnhoTxRl3LmeitIlAG+zCpiT4J+se2eG27ep4aUA
bYuO0L+KBosY6DcvEbR9TqkYLPgJuEPi/6SQr9bBOldEspXXAv9msyCB4VMhVU=
- job:
name: system-config-build-image
parent: opendev-build-docker-image
dependencies: opendev-buildset-registry
abstract: true
- job:
name: system-config-upload-image
parent: opendev-upload-docker-image
dependencies: opendev-buildset-registry
abstract: true
secrets:
name: docker_credentials
secret: system-config-dockerhub
pass-to-parent: true
- job:
name: system-config-promote-image
parent: opendev-promote-docker-image
abstract: true
secrets:
name: docker_credentials
secret: system-config-dockerhub
pass-to-parent: true
# Jinja-init jobs
- job:
name: system-config-build-image-jinja-init
description: Build a jinja-init image.
parent: system-config-build-image
vars: &jinja-init_vars
docker_images:
- context: docker/jinja-init
target: jinja-init
repository: opendevorg/jinja-init
files: &jinja-init_files
- docker/jinja-init/.*
- job:
name: system-config-upload-image-jinja-init
description: Build and upload a jinja-init image.
parent: system-config-upload-image
vars: *jinja-init_vars
files: *jinja-init_files
- job:
name: system-config-promote-image-jinja-init
description: Promote a previously published jinja-init image to latest.
parent: system-config-promote-image
vars: *jinja-init_vars
files: *jinja-init_files
# Gitea-init jobs
- job:
name: system-config-build-image-gitea-init
description: Build a gitea-init image.
parent: system-config-build-image
vars: &gitea-init_vars
docker_images:
- context: docker/gitea-init
target: gitea-init
repository: opendevorg/gitea-init
files: &gitea-init_files
- docker/gitea-init/.*
- job:
name: system-config-upload-image-gitea-init
description: Build and upload a gitea-init image.
parent: system-config-upload-image
vars: *gitea-init_vars
files: *gitea-init_files
- job:
name: system-config-promote-image-gitea-init
description: Promote a previously published gitea-init image to latest.
parent: system-config-promote-image
vars: *gitea-init_vars
files: *gitea-init_files
# Gitea jobs
- job:
name: system-config-build-image-gitea
description: Build a gitea image.
parent: system-config-build-image
vars: &gitea_vars
docker_images:
- context: docker/gitea
target: gitea
repository: opendevorg/gitea
- context: docker/gitea
target: gitea-openssh
repository: opendevorg/gitea-openssh
# Duplicate in the run-gitea job
files: &gitea_files
- docker/gitea/.*
- job:
name: system-config-upload-image-gitea
description: Build and upload a gitea image.
parent: system-config-upload-image
vars: *gitea_vars
files: *gitea_files
- job:
name: system-config-promote-image-gitea
description: Promote a previously published gitea image to latest.
parent: system-config-promote-image
vars: *gitea_vars
files: *gitea_files
# Haproxy-statsd jobs
- job:
name: system-config-build-image-haproxy-statsd
description: Build a haproxy-statsd image.
parent: system-config-build-image
vars: &haproxy-statsd_vars
docker_images:
- context: docker/haproxy-statsd
repository: opendevorg/haproxy-statsd
# Duplicate in the run-gitea job
files: &haproxy-statsd_files
- docker/haproxy-statsd/.*
- job:
name: system-config-upload-image-haproxy-statsd
description: Build and upload a haproxy-statsd image.
parent: system-config-upload-image
vars: *haproxy-statsd_vars
files: *haproxy-statsd_files
- job:
name: system-config-promote-image-haproxy-statsd
description: Promote a previously published haproxy-statsd image to latest.
parent: system-config-promote-image
vars: *haproxy-statsd_vars
files: *haproxy-statsd_files
# Gerrit jobs
- job:
name: system-config-build-image-gerrit
description: Build a gerrit image.
parent: system-config-build-image
pre-run: playbooks/zuul/gerrit/repos.yaml
required-projects: &gerrit_projects
- name: gerrit.googlesource.com/gerrit
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/commit-message-length-validator
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/download-commands
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/hooks
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/its-base
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/its-storyboard
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/javamelody
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/replication
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/reviewnotes
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/singleusergroup
override-checkout: stable-2.15
vars: &gerrit_vars
docker_images:
- context: docker/gerrit
target: gerrit
repository: opendevorg/gerrit
path: /home/zuul/src/gerrit.googlesource.com/gerrit
tags:
- 2.15
build_args:
- BAZEL_OPTS="--local_resources=4096,2.0,1.0"
files: &gerrit_files
- docker/gerrit/.*
- playbooks/zuul/gerrit/.*
- job:
name: system-config-upload-image-gerrit
description: Build and upload a gerrit image.
parent: system-config-upload-image
pre-run: playbooks/zuul/gerrit/repos.yaml
required-projects: *gerrit_projects
vars: *gerrit_vars
files: *gerrit_files
- job:
name: system-config-promote-image-gerrit
description: Promote a previously published gerrit image to latest.
parent: system-config-promote-image
vars: *gerrit_vars
files: *gerrit_files
# python-builder jobs
- job:
name: system-config-build-image-python-builder
description: Build a python-builder image.
parent: system-config-build-image
vars: &python-builder_vars
docker_images:
- context: docker/python-builder
repository: opendevorg/python-builder
files: &python-builder_files
- docker/python-builder/.*
- job:
name: system-config-upload-image-python-builder
description: Build and upload a python-builder image.
parent: system-config-upload-image
vars: *python-builder_vars
files: *python-builder_files
- job:
name: system-config-promote-image-python-builder
description: Promote a previously published python-builder image to latest.
parent: system-config-promote-image
vars: *python-builder_vars
files: *python-builder_files
# python-base jobs
- job:
name: system-config-build-image-python-base
description: Build a python-base image.
parent: system-config-build-image
vars: &python-base_vars
docker_images:
- context: docker/python-base
repository: opendevorg/python-base
files: &python-base_files
- docker/python-base/.*
- job:
name: system-config-upload-image-python-base
description: Build and upload a python-base image.
parent: system-config-upload-image
vars: *python-base_vars
files: *python-base_files
- job:
name: system-config-promote-image-python-base
description: Promote a previously published python-base image to latest.
parent: system-config-promote-image
vars: *python-base_vars
files: *python-base_files
# Role integration jobs. These test the top-level generic roles/*
# under Zuul. The range of platforms should be the same as those for
# openstack-zuul-jobs.
#
# NOTE(ianw): 2018-08 have left off Fedora & OpenSUSE because
# no roles currently run on them.
- job:
name: system-config-zuul-role-integration
description: |
Test roles provided by system-config with Zuul
abstract: true
parent: base
run: roles-test/base.yaml
files:
- roles/.*
- job:
name: system-config-zuul-role-integration-xenial
parent: system-config-zuul-role-integration
nodeset:
nodes:
- name: base
label: ubuntu-xenial
- name: puppet4
label: ubuntu-xenial
groups:
- name: puppet3
nodes:
- base
- name: openafs
nodes:
- base
- job:
name: system-config-zuul-role-integration-bionic
parent: system-config-zuul-role-integration
nodeset:
nodes:
- name: base
label: ubuntu-bionic
groups:
- name: puppet5
nodes:
- base
- name: openafs
nodes:
- base
- job:
name: system-config-zuul-role-integration-debian-stable
parent: system-config-zuul-role-integration
nodeset:
nodes:
- name: base
label: debian-stretch
groups:
- name: openafs
nodes:
- base
- project-template:
name: system-config-zuul-role-integration
check:
jobs:
- system-config-zuul-role-integration-xenial
- system-config-zuul-role-integration-bionic
- system-config-zuul-role-integration-debian-stable
gate:
jobs:
- system-config-zuul-role-integration-xenial
- system-config-zuul-role-integration-bionic
- system-config-zuul-role-integration-debian-stable
- job:
name: system-config-run
description: |
Run the "base" playbook for system-config hosts.
This is a parent job designed to be inherited.
abstract: true
pre-run: playbooks/zuul/run-base-pre.yaml
run: playbooks/zuul/run-base.yaml
post-run: playbooks/zuul/run-base-post.yaml
vars:
install_ansible_ara_enable: true
zuul_copy_output: "{{ copy_output | combine(host_copy_output | default({})) }}"
stage_dir: "{{ ansible_user_dir }}/zuul-output"
copy_output:
'/var/log/syslog': logs_txt
'/var/log/messages': logs_txt
'/var/log/docker': logs
host-vars:
bridge.openstack.org:
host_copy_output:
'{{ zuul.project.src_dir }}/junit.xml': logs
- job:
name: system-config-run-base
parent: system-config-run
description: |
Run the "base" playbook on each of the node types
currently in use.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: trusty
label: ubuntu-trusty
- name: xenial
label: ubuntu-xenial
- name: bionic
label: ubuntu-bionic
host-vars:
trusty:
ansible_python_interpreter: python2
files:
- .zuul.yaml
- playbooks/.*
- roles/.*
- testinfra/.*
- job:
name: system-config-run-base-ansible-devel
parent: system-config-run-base
description: |
Run the base playbook with the latest ansible
required-projects:
- name: github.com/ansible/ansible
override-checkout: devel
- name: github.com/philpep/testinfra
- name: openstack/openstacksdk
- name: recordsansible/ara
# NOTE(ianw): 2019-06-11 pinned to stable branch until we handle 1.0
# http://lists.openstack.org/pipermail/openstack-infra/2019-June/006400.html
override-checkout: stable/0.x
vars:
bridge_ansible_name: '{{ ansible_user_dir}}/src/github.com/ansible/ansible'
bridge_ansible_version: null
bridge_openstacksdk_name: '{{ ansible_user_dir }}/src/opendev.org/openstack/openstacksdk'
bridge_openstacksdk_version: null
bridge_ara_name: '{{ ansible_user_dir}}/src/opendev.org/recordsansible/ara'
bridge_ara_version: null
- job:
name: system-config-run-eavesdrop
parent: system-config-run
description: |
Run the playbook for an eavesdrop server.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: eavesdrop01.openstack.org
label: ubuntu-xenial
files:
- .zuul.yaml
- playbooks/group_vars/eavesdrop.yaml
- testinfra/test_eavesdrop.py
vars:
run_playbooks:
- playbooks/remote_puppet_else.yaml
- job:
name: system-config-run-letsencrypt
parent: system-config-run
description: |
Run the playbook for letsencrypt key acquisition
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns-letsencrypt.opendev.org
label: ubuntu-bionic
- name: letsencrypt01.opendev.org
label: ubuntu-bionic
- name: letsencrypt02.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-nameserver.yaml
- playbooks/service-letsencrypt.yaml
host-vars:
letsencrypt01.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
letsencrypt02.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
files:
- .zuul.yaml
- playbooks/group_vars/letsencrypt.yaml
- playbooks/roles/letsencrypt.*
- job:
name: system-config-run-lists
parent: system-config-run
description: |
Run the playbook for a list server.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: lists.openstack.org
label: ubuntu-xenial
files:
- .zuul.yaml
- modules/openstack_project/manifests/lists.pp
- playbooks/host_vars/lists.openstack.org.yaml
- roles/exim
vars:
run_playbooks:
- playbooks/remote_puppet_else.yaml
- job:
name: system-config-run-nodepool
parent: system-config-run
description: |
Run the playbook for nodepool.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: nl01.openstack.org
label: ubuntu-xenial
- name: nb01.openstack.org
label: ubuntu-xenial
vars:
run_playbooks:
- playbooks/service-nodepool.yaml
- playbooks/remote_puppet_else.yaml
files:
- .zuul.yaml
- playbooks/group_vars/nodepool.yaml
- playbooks/group_vars/nodepool-builder.yaml
- playbooks/group_vars/nodepool-launcher.yaml
- playbooks/roles/configure-openstacksdk/
- playbooks/templates/clouds/
- testinfra/test_nodepool.py
- job:
name: system-config-run-dns
parent: system-config-run
description: |
Run the playbook for dns.
required-projects:
- opendev/zone-opendev.org
- opendev/zone-zuul-ci.org
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns1.opendev.org
label: ubuntu-bionic
- name: ns1.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-nameserver.yaml
host-vars:
adns1.opendev.org:
host_copy_output:
'/etc/bind/named.conf': logs
'/var/lib/bind/zones': logs
files:
- .zuul.yaml
- playbooks/group_vars/adns.yaml
- playbooks/group_vars/dns.yaml
- ^playbooks/host_vars/(ad)?ns\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/adns.yaml.j2
- playbooks/zuul/templates/group_vars/ns.yaml.j2
- playbooks/roles/master-nameserver/
- playbooks/roles/nameserver/
- testinfra/test_adns.py
- testinfra/test_ns.py
- job:
name: system-config-run-mirror
parent: system-config-run
description: |
Run the playbook for a mirror node
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: mirror01.openafs.provider.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-mirror.yaml
files:
- .zuul.yaml
- roles/
- playbooks/roles/mirror/
- playbooks/roles/letsencrypt.*
- playbooks/service-letsencrypt.yaml
- playbooks/service-mirror.yaml
- testinfra/test_mirror.py
host-vars:
mirror.region.provider.opendev.org:
host_copy_output:
'/var/log/apache2/': logs
- job:
name: system-config-run-mirror-update
parent: system-config-run
description: |
Run the playbook for a mirror update node
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: mirror-update01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-mirror-update.yaml
files:
- .zuul.yaml
- roles/
- playbooks/roles/mirror-update/
- playbooks/service-mirror-update.yaml
- testinfra/test_mirror-update.py
- job:
name: system-config-run-docker-registry
parent: system-config-run
description: |
Run the playbook for the docker registry.
dependencies: opendev-buildset-registry
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: insecure-ci-registry01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-registry.yaml
host-vars:
insecure-ci-registry01.opendev.org:
host_copy_output:
'/var/registry/auth': logs
'/var/registry/certs': logs
files:
- .zuul.yaml
- playbooks/group_vars/registry.yaml
- ^playbooks/host_vars/insecure-ci-registry\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/registry.yaml.j2
- playbooks/roles/registry/
- testinfra/test_registry.py
- job:
name: system-config-run-gitea
parent: system-config-run
description: |
Run the playbook for the gitea servers.
dependencies: opendev-buildset-registry
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: gitea-lb01.opendev.org
label: ubuntu-bionic
- name: gitea01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-gitea-lb.yaml
run_test_playbook: playbooks/test-gitea.yaml
host-vars:
gitea01.opendev.org:
host_copy_output:
'/var/gitea/conf': logs
'/var/gitea/certs': logs
'/var/gitea/logs': logs
gitea-lb01.opendev.org:
host_copy_output:
'/var/haproxy/etc': logs
files:
- .zuul.yaml
- playbooks/group_vars/gitea.yaml
- playbooks/group_vars/gitea-lb.yaml
- playbooks/zuul/templates/group_vars/gitea.yaml.j2
- playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
- playbooks/roles/gitea/
- playbooks/roles/gitea-git-repos/
- playbooks/roles/haproxy/
- testinfra/test_gitea.py
- testinfra/test_gitea_lb.py
# From gitea_files -- If we rebuild the image, we want to run
# this job as well.
- docker/gitea/.*
# From haproxy-statsd_files -- If we rebuild the image, we want
# to run this job as well.
- docker/haproxy-statsd/.*
- job:
name: system-config-run-zuul-preview
parent: system-config-run
description: |
Run the playbook for the docker registry.
dependencies: opendev-buildset-registry
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zp01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-zuul.yaml
files:
- .zuul.yaml
- playbooks/roles/zuul-preview/
- testinfra/test_zuul_preview.py
- job:
name: infra-prod-playbook
description: |
Run specified playbook against productions hosts.
This is a parent job designed to be inherited to enabled
CD deployment of our infrastructure. Set playbook_name to
specify the playbook relative to
bridge.openstack.org:/opt/system-config/playbooks
abstract: true
run: playbooks/zuul/run-production-playbook.yaml
vars:
ansible_forks: 5
nodeset:
nodes: []
- job:
name: infra-prod-zuul_reconfigure
parent: infra-prod-playbook
description: |
Perform a full Zuul reconfiguration.
This reloads the tenant config file and clears all configuration
caches.
vars:
playbook_name: zuul_reconfigure.yaml
- project:
templates:
- system-config-zuul-role-integration
- infra-puppet-apply-jobs
- infra-logstash-filter-jobs
- publish-tox-docs-infra
check:
jobs:
- opendev-buildset-registry
- tox-linters
- puppet-beaker-rspec-puppet-4-infra-system-config
- system-config-run-base
- system-config-run-base-ansible-devel:
voting: false
- system-config-run-dns
- system-config-run-eavesdrop
- system-config-run-lists
- system-config-run-nodepool
- system-config-run-mirror
- system-config-run-mirror-update
- system-config-run-docker-registry
- system-config-run-gitea:
dependencies:
- name: system-config-build-image-gitea
soft: true
- name: system-config-build-image-haproxy-statsd
soft: true
- system-config-run-zuul-preview
- system-config-run-letsencrypt
- system-config-build-image-jinja-init
- system-config-build-image-gitea-init
- system-config-build-image-gitea
- system-config-build-image-gerrit
- system-config-build-image-haproxy-statsd
- system-config-build-image-python-base
- system-config-build-image-python-builder
gate:
jobs:
- opendev-buildset-registry
- tox-linters
- puppet-beaker-rspec-puppet-4-infra-system-config
- system-config-run-base
- system-config-run-dns
- system-config-run-eavesdrop
- system-config-run-lists
- system-config-run-nodepool
- system-config-run-mirror
- system-config-run-mirror-update
- system-config-run-docker-registry
- system-config-run-gitea:
dependencies:
- name: system-config-upload-image-gitea
soft: true
- name: system-config-upload-image-haproxy-statsd
soft: true
- system-config-run-zuul-preview
- system-config-run-letsencrypt
- system-config-upload-image-jinja-init
- system-config-upload-image-gitea-init
- system-config-upload-image-gitea
- system-config-upload-image-gerrit
- system-config-upload-image-haproxy-statsd
- system-config-upload-image-python-base
- system-config-upload-image-python-builder
promote:
jobs:
- system-config-promote-image-jinja-init
- system-config-promote-image-gitea-init
- system-config-promote-image-gitea
- system-config-promote-image-gerrit
- system-config-promote-image-haproxy-statsd
- system-config-promote-image-python-base
- system-config-promote-image-python-builder