system-config/.zuul.yaml
Monty Taylor 943f66e3e6 Build docker images of gerrit
To provide a stepwise upgrade path from 2.13 running directly to
2.15 in a container, make a container image containing the war we're
using currently. This should let us make a change to how we run the
war without changing the war at all, and then update the war.

Instead of trying to make a clean build for gerrit 2.13 inside of a
builder image, just have it wget the already built wars and jars we
have.

There are pieces of this that duplicate what's being done in puppet,
but in this context it's not immediately clear these are important to
do. However, it's also not clear they're a bad idea.

The gerrit 2.15 build needs a newer bazel. Looking at the CI scripts
that are used by gerrithub, we find that they use bazel 0.26.1
and nodesource v10. Use the bazel image published by google to get
a bazel builder image.

Set gerrit uid/git to 3000 in both images to match the existing
directory ownership so that bindmounting doesn't face permissions
problems.

Change-Id: I3533f01c0859ed50640dcfd98023994c5867c056
2019-07-24 04:40:28 -04:00

779 lines
24 KiB
YAML

- job:
name: puppet-beaker-rspec-puppet-4-infra-system-config
parent: puppet-beaker-rspec-puppet-4-infra
vars:
project_src_dir: "{{ zuul.project.src_dir }}/modules/openstack_project"
# Image building jobs
- secret:
name: system-config-dockerhub
data:
username: opendevzuul
password: !encrypted/pkcs1-oaep
- FF2mwsdYo4b9QI/kEmeFdZS32I+OJUac2xy4JEQJaIevRK9E3rQ/11cBn7xX6zelA/htY
hENMM1gGTYO8BKgTVNijFfCiVCTYFJ9efsOrSeMdj2aLcq/OkZFeq8gMHAZDv+3qaiggq
hbDmldgSouNahhJByWMsDjYqnItfR8w1dYisNQBkDEJdNW5+Cin5WgHRYv2skui9pjumU
vWHjwLbITLEjb0hwVErggAzvED4x0UjNEMfb6KCpoXeKdlCaI8h1DXVD9RdQ4s1WvUCro
xwl4AIUyCmdtXbqW787PQ4j8BgdOi8RD40ZTSxpoV/bwT+/NrP3Jj2KVYdEKQJimzXnwL
goDclapu9aFrAaqwWSnWCRc7GQptSZ7DBXmA7MsNinBclWHX/chX5k9Q1+XifAhmKJZqP
ovkzEGtmUjTQZFjvuCtFxIWZEGVtGHDmgEkzevgVCeuHDTeIDlkAvA3+TT+/A70zyYdbP
EHOTSSjCU6RSlJxAXjNb5pzKjpeT7PyX2f4/8Zn3JnwTlfPJ0C7qdEYjQ54hbsKynjrpQ
7f6NN9pAeQS1pid5J7oH4f7YaKM78pnhoTxRl3LmeitIlAG+zCpiT4J+se2eG27ep4aUA
bYuO0L+KBosY6DcvEbR9TqkYLPgJuEPi/6SQr9bBOldEspXXAv9msyCB4VMhVU=
- job:
name: system-config-build-image
parent: opendev-build-docker-image
dependencies: opendev-buildset-registry
abstract: true
- job:
name: system-config-upload-image
parent: opendev-upload-docker-image
dependencies: opendev-buildset-registry
abstract: true
secrets:
name: docker_credentials
secret: system-config-dockerhub
pass-to-parent: true
- job:
name: system-config-promote-image
parent: opendev-promote-docker-image
abstract: true
secrets:
name: docker_credentials
secret: system-config-dockerhub
pass-to-parent: true
# Jinja-init jobs
- job:
name: system-config-build-image-jinja-init
description: Build a jinja-init image.
parent: system-config-build-image
vars: &jinja-init_vars
docker_images:
- context: docker/jinja-init
target: jinja-init
repository: opendevorg/jinja-init
files: &jinja-init_files
- docker/jinja-init/.*
- job:
name: system-config-upload-image-jinja-init
description: Build and upload a jinja-init image.
parent: system-config-upload-image
vars: *jinja-init_vars
files: *jinja-init_files
- job:
name: system-config-promote-image-jinja-init
description: Promote a previously published jinja-init image to latest.
parent: system-config-promote-image
vars: *jinja-init_vars
files: *jinja-init_files
# Gitea-init jobs
- job:
name: system-config-build-image-gitea-init
description: Build a gitea-init image.
parent: system-config-build-image
vars: &gitea-init_vars
docker_images:
- context: docker/gitea-init
target: gitea-init
repository: opendevorg/gitea-init
files: &gitea-init_files
- docker/gitea-init/.*
- job:
name: system-config-upload-image-gitea-init
description: Build and upload a gitea-init image.
parent: system-config-upload-image
vars: *gitea-init_vars
files: *gitea-init_files
- job:
name: system-config-promote-image-gitea-init
description: Promote a previously published gitea-init image to latest.
parent: system-config-promote-image
vars: *gitea-init_vars
files: *gitea-init_files
# Gitea jobs
- job:
name: system-config-build-image-gitea
description: Build a gitea image.
parent: system-config-build-image
vars: &gitea_vars
docker_images:
- context: docker/gitea
target: gitea
repository: opendevorg/gitea
- context: docker/gitea
target: gitea-openssh
repository: opendevorg/gitea-openssh
# Duplicate in the run-gitea job
files: &gitea_files
- docker/gitea/.*
- job:
name: system-config-upload-image-gitea
description: Build and upload a gitea image.
parent: system-config-upload-image
vars: *gitea_vars
files: *gitea_files
- job:
name: system-config-promote-image-gitea
description: Promote a previously published gitea image to latest.
parent: system-config-promote-image
vars: *gitea_vars
files: *gitea_files
# Haproxy-statsd jobs
- job:
name: system-config-build-image-haproxy-statsd
description: Build a haproxy-statsd image.
parent: system-config-build-image
vars: &haproxy-statsd_vars
docker_images:
- context: docker/haproxy-statsd
repository: opendevorg/haproxy-statsd
# Duplicate in the run-gitea job
files: &haproxy-statsd_files
- docker/haproxy-statsd/.*
- job:
name: system-config-upload-image-haproxy-statsd
description: Build and upload a haproxy-statsd image.
parent: system-config-upload-image
vars: *haproxy-statsd_vars
files: *haproxy-statsd_files
- job:
name: system-config-promote-image-haproxy-statsd
description: Promote a previously published haproxy-statsd image to latest.
parent: system-config-promote-image
vars: *haproxy-statsd_vars
files: *haproxy-statsd_files
# Gerrit jobs
- job:
name: system-config-build-image-gerrit
description: Build a gerrit image.
parent: system-config-build-image
pre-run: playbooks/zuul/gerrit/repos.yaml
required-projects: &gerrit_projects
- name: gerrit.googlesource.com/gerrit
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/commit-message-length-validator
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/download-commands
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/hooks
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/its-base
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/its-storyboard
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/javamelody
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/replication
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/reviewnotes
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/singleusergroup
override-checkout: stable-2.15
vars: &gerrit_vars
docker_images:
- context: docker/gerrit/2.15
repository: opendevorg/gerrit
path: /home/zuul/src/gerrit.googlesource.com/gerrit
tags:
- 2.15
# The 2.13 image doesn't build from source, but from existing war file
- context: docker/gerrit/2.13
repository: opendevorg/gerrit
path: /home/zuul/src/opendev.org/opendev/system-config
tags:
- 2.13
files: &gerrit_files
- docker/gerrit/.*
- playbooks/zuul/gerrit/.*
- job:
name: system-config-upload-image-gerrit
description: Build and upload a gerrit image.
parent: system-config-upload-image
pre-run: playbooks/zuul/gerrit/repos.yaml
required-projects: *gerrit_projects
vars: *gerrit_vars
files: *gerrit_files
- job:
name: system-config-promote-image-gerrit
description: Promote a previously published gerrit image to latest.
parent: system-config-promote-image
vars: *gerrit_vars
files: *gerrit_files
# python-builder jobs
- job:
name: system-config-build-image-python-builder
description: Build a python-builder image.
parent: system-config-build-image
vars: &python-builder_vars
docker_images:
- context: docker/python-builder
repository: opendevorg/python-builder
files: &python-builder_files
- docker/python-builder/.*
- job:
name: system-config-upload-image-python-builder
description: Build and upload a python-builder image.
parent: system-config-upload-image
vars: *python-builder_vars
files: *python-builder_files
- job:
name: system-config-promote-image-python-builder
description: Promote a previously published python-builder image to latest.
parent: system-config-promote-image
vars: *python-builder_vars
files: *python-builder_files
# python-base jobs
- job:
name: system-config-build-image-python-base
description: Build a python-base image.
parent: system-config-build-image
vars: &python-base_vars
docker_images:
- context: docker/python-base
repository: opendevorg/python-base
files: &python-base_files
- docker/python-base/.*
- job:
name: system-config-upload-image-python-base
description: Build and upload a python-base image.
parent: system-config-upload-image
vars: *python-base_vars
files: *python-base_files
- job:
name: system-config-promote-image-python-base
description: Promote a previously published python-base image to latest.
parent: system-config-promote-image
vars: *python-base_vars
files: *python-base_files
# Role integration jobs. These test the top-level generic roles/*
# under Zuul. The range of platforms should be the same as those for
# openstack-zuul-jobs.
#
# NOTE(ianw): 2018-08 have left off Fedora & OpenSUSE because
# no roles currently run on them.
- job:
name: system-config-zuul-role-integration
description: |
Test roles provided by system-config with Zuul
abstract: true
parent: base
run: roles-test/base.yaml
files:
- roles/.*
- job:
name: system-config-zuul-role-integration-xenial
parent: system-config-zuul-role-integration
nodeset:
nodes:
- name: base
label: ubuntu-xenial
- name: puppet4
label: ubuntu-xenial
groups:
- name: puppet3
nodes:
- base
- name: openafs
nodes:
- base
- job:
name: system-config-zuul-role-integration-bionic
parent: system-config-zuul-role-integration
nodeset:
nodes:
- name: base
label: ubuntu-bionic
groups:
- name: puppet5
nodes:
- base
- name: openafs
nodes:
- base
- job:
name: system-config-zuul-role-integration-debian-stable
parent: system-config-zuul-role-integration
nodeset:
nodes:
- name: base
label: debian-stretch
groups:
- name: openafs
nodes:
- base
- project-template:
name: system-config-zuul-role-integration
check:
jobs:
- system-config-zuul-role-integration-xenial
- system-config-zuul-role-integration-bionic
- system-config-zuul-role-integration-debian-stable
gate:
jobs:
- system-config-zuul-role-integration-xenial
- system-config-zuul-role-integration-bionic
- system-config-zuul-role-integration-debian-stable
- job:
name: system-config-run
description: |
Run the "base" playbook for system-config hosts.
This is a parent job designed to be inherited.
abstract: true
pre-run: playbooks/zuul/run-base-pre.yaml
run: playbooks/zuul/run-base.yaml
post-run: playbooks/zuul/run-base-post.yaml
vars:
install_ansible_ara_enable: true
zuul_copy_output: "{{ copy_output | combine(host_copy_output | default({})) }}"
stage_dir: "{{ ansible_user_dir }}/zuul-output"
copy_output:
'/var/log/syslog': logs_txt
'/var/log/messages': logs_txt
'/var/log/docker': logs
host-vars:
bridge.openstack.org:
host_copy_output:
'{{ zuul.project.src_dir }}/junit.xml': logs
- job:
name: system-config-run-base
parent: system-config-run
description: |
Run the "base" playbook on each of the node types
currently in use.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: trusty
label: ubuntu-trusty
- name: xenial
label: ubuntu-xenial
- name: bionic
label: ubuntu-bionic
files:
- playbooks/.*
- roles/.*
- testinfra/.*
- job:
name: system-config-run-base-ansible-devel
parent: system-config-run-base
description: |
Run the base playbook with the latest ansible
required-projects:
- name: github.com/ansible/ansible
override-checkout: devel
- name: github.com/philpep/testinfra
- name: openstack/openstacksdk
- name: recordsansible/ara
# NOTE(ianw): 2019-06-11 pinned to stable branch until we handle 1.0
# http://lists.openstack.org/pipermail/openstack-infra/2019-June/006400.html
override-checkout: stable/0.x
vars:
bridge_ansible_name: '{{ ansible_user_dir}}/src/github.com/ansible/ansible'
bridge_ansible_version: null
bridge_openstacksdk_name: '{{ ansible_user_dir }}/src/opendev.org/openstack/openstacksdk'
bridge_openstacksdk_version: null
bridge_ara_name: '{{ ansible_user_dir}}/src/opendev.org/recordsansible/ara'
bridge_ara_version: null
- job:
name: system-config-run-eavesdrop
parent: system-config-run
description: |
Run the playbook for an eavesdrop server.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: eavesdrop01.openstack.org
label: ubuntu-xenial
files:
- playbooks/group_vars/eavesdrop.yaml
- testinfra/test_eavesdrop.py
vars:
run_playbooks:
- playbooks/remote_puppet_else.yaml
- job:
name: system-config-run-letsencrypt
parent: system-config-run
description: |
Run the playbook for letsencrypt key acquisition
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns-letsencrypt.opendev.org
label: ubuntu-bionic
- name: letsencrypt01.opendev.org
label: ubuntu-bionic
- name: letsencrypt02.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-nameserver.yaml
- playbooks/service-letsencrypt.yaml
host-vars:
letsencrypt01.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
letsencrypt02.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
files:
- playbooks/group_vars/letsencrypt.yaml
- playbooks/roles/letsencrypt.*
- job:
name: system-config-run-lists
parent: system-config-run
description: |
Run the playbook for a list server.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: lists.openstack.org
label: ubuntu-xenial
files:
- modules/openstack_project/manifests/lists.pp
- playbooks/host_vars/lists.openstack.org.yaml
- roles/exim
vars:
run_playbooks:
- playbooks/remote_puppet_else.yaml
- job:
name: system-config-run-nodepool
parent: system-config-run
description: |
Run the playbook for nodepool.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: nl01.openstack.org
label: ubuntu-xenial
- name: nb01.openstack.org
label: ubuntu-xenial
vars:
run_playbooks:
- playbooks/service-nodepool.yaml
- playbooks/remote_puppet_else.yaml
files:
- playbooks/group_vars/nodepool.yaml
- playbooks/group_vars/nodepool-builder.yaml
- playbooks/group_vars/nodepool-launcher.yaml
- playbooks/roles/configure-openstacksdk/
- playbooks/templates/clouds/
- testinfra/test_nodepool.py
- job:
name: system-config-run-dns
parent: system-config-run
description: |
Run the playbook for dns.
required-projects:
- opendev/zone-opendev.org
- opendev/zone-zuul-ci.org
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns1.opendev.org
label: ubuntu-bionic
- name: ns1.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-nameserver.yaml
host-vars:
adns1.opendev.org:
host_copy_output:
'/etc/bind/named.conf': logs
'/var/lib/bind/zones': logs
files:
- playbooks/group_vars/adns.yaml
- playbooks/group_vars/dns.yaml
- ^playbooks/host_vars/(ad)?ns\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/adns.yaml.j2
- playbooks/zuul/templates/group_vars/ns.yaml.j2
- playbooks/roles/master-nameserver/
- playbooks/roles/nameserver/
- testinfra/test_adns.py
- testinfra/test_ns.py
- job:
name: system-config-run-mirror
parent: system-config-run
description: |
Run the playbook for a mirror node
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: mirror01.openafs.provider.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-mirror.yaml
files:
- roles/
- playbooks/roles/mirror/
- playbooks/roles/letsencrypt.*
- playbooks/service-letsencrypt.yaml
- playbooks/service-mirror.yaml
- testinfra/test_mirror.py
host-vars:
mirror.region.provider.opendev.org:
host_copy_output:
'/var/log/apache2/': logs
- job:
name: system-config-run-mirror-update
parent: system-config-run
description: |
Run the playbook for a mirror update node
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: mirror-update01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-mirror-update.yaml
files:
- roles/
- playbooks/roles/mirror-update/
- playbooks/service-mirror-update.yaml
- testinfra/test_mirror-update.py
- job:
name: system-config-run-docker-registry
parent: system-config-run
description: |
Run the playbook for the docker registry.
dependencies: opendev-buildset-registry
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: insecure-ci-registry01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-registry.yaml
host-vars:
insecure-ci-registry01.opendev.org:
host_copy_output:
'/var/registry/auth': logs
'/var/registry/certs': logs
files:
- playbooks/group_vars/registry.yaml
- ^playbooks/host_vars/insecure-ci-registry\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/registry.yaml.j2
- playbooks/roles/registry/
- testinfra/test_registry.py
- job:
name: system-config-run-gitea
parent: system-config-run
description: |
Run the playbook for the gitea servers.
dependencies: opendev-buildset-registry
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: gitea-lb01.opendev.org
label: ubuntu-bionic
- name: gitea01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-gitea-lb.yaml
- playbooks/remote_puppet_git.yaml
run_test_playbook: playbooks/test-gitea.yaml
host-vars:
gitea01.opendev.org:
host_copy_output:
'/var/gitea/conf': logs
'/var/gitea/certs': logs
'/var/gitea/logs': logs
gitea-lb01.opendev.org:
host_copy_output:
'/var/haproxy/etc': logs
files:
- playbooks/group_vars/gitea.yaml
- playbooks/group_vars/gitea-lb.yaml
- playbooks/zuul/templates/group_vars/gitea.yaml.j2
- playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
- playbooks/roles/gitea/
- playbooks/roles/gitea-git-repos/
- playbooks/roles/haproxy/
- testinfra/test_gitea.py
- testinfra/test_gitea_lb.py
# From gitea_files -- If we rebuild the image, we want to run
# this job as well.
- docker/gitea/.*
# From haproxy-statsd_files -- If we rebuild the image, we want
# to run this job as well.
- docker/haproxy-statsd/.*
- job:
name: system-config-run-zuul-preview
parent: system-config-run
description: |
Run the playbook for the docker registry.
dependencies: opendev-buildset-registry
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zp01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-zuul.yaml
files:
- playbooks/roles/zuul-preview/
- testinfra/test_zuul_preview.py
- job:
name: infra-prod-playbook
description: |
Run specified playbook against productions hosts.
This is a parent job designed to be inherited to enabled
CD deployment of our infrastructure. Set playbook_name to
specify the playbook relative to
bridge.openstack.org:/opt/system-config/playbooks
abstract: true
run: playbooks/zuul/run-production-playbook.yaml
vars:
ansible_forks: 5
nodeset:
nodes: []
- job:
name: infra-prod-zuul_reconfigure
parent: infra-prod-playbook
description: |
Perform a full Zuul reconfiguration.
This reloads the tenant config file and clears all configuration
caches.
vars:
playbook_name: zuul_reconfigure.yaml
- project:
templates:
- system-config-zuul-role-integration
- infra-puppet-apply-jobs
- infra-logstash-filter-jobs
- publish-tox-docs-infra
check:
jobs:
- opendev-buildset-registry
- tox-linters
- puppet-beaker-rspec-puppet-4-infra-system-config
- system-config-run-base
- system-config-run-base-ansible-devel:
voting: false
- system-config-run-dns
- system-config-run-eavesdrop
- system-config-run-lists
- system-config-run-nodepool
- system-config-run-mirror
- system-config-run-mirror-update
- system-config-run-docker-registry
- system-config-run-gitea:
dependencies:
- name: system-config-build-image-gitea
soft: true
- name: system-config-build-image-haproxy-statsd
soft: true
- system-config-run-zuul-preview
- system-config-run-letsencrypt
- system-config-build-image-jinja-init
- system-config-build-image-gitea-init
- system-config-build-image-gitea
- system-config-build-image-gerrit
- system-config-build-image-haproxy-statsd
- system-config-build-image-python-base
- system-config-build-image-python-builder
gate:
jobs:
- opendev-buildset-registry
- tox-linters
- puppet-beaker-rspec-puppet-4-infra-system-config
- system-config-run-base
- system-config-run-dns
- system-config-run-eavesdrop
- system-config-run-lists
- system-config-run-nodepool
- system-config-run-mirror
- system-config-run-mirror-update
- system-config-run-docker-registry
- system-config-run-gitea:
dependencies:
- name: system-config-upload-image-gitea
soft: true
- name: system-config-upload-image-haproxy-statsd
soft: true
- system-config-run-zuul-preview
- system-config-run-letsencrypt
- system-config-upload-image-jinja-init
- system-config-upload-image-gitea-init
- system-config-upload-image-gitea
- system-config-upload-image-gerrit
- system-config-upload-image-haproxy-statsd
- system-config-upload-image-python-base
- system-config-upload-image-python-builder
promote:
jobs:
- system-config-promote-image-jinja-init
- system-config-promote-image-gitea-init
- system-config-promote-image-gitea
- system-config-promote-image-gerrit
- system-config-promote-image-haproxy-statsd
- system-config-promote-image-python-base
- system-config-promote-image-python-builder