system-config/.zuul.yaml
Ian Wienand afd907c16d letsencrypt support
This change contains the roles and testing for deploying certificates
on hosts using letsencrypt with domain authentication.

From a top level, the process is implemented in the roles as follows:

1) letsencrypt-acme-sh-install

   This role installs the acme.sh tool on hosts in the letsencrypt
   group, along with a small custom driver script to help parse output
   that is used by later roles.

2) letsencrypt-request-certs

   This role runs on each host, and reads a host variable describing
   the certificates required.  It uses the acme.sh tool (via the
   driver) to request the certificates from letsencrypt.  It populates
   a global Ansible variable with the authentication TXT records
   required.

   If the certificate exists on the host and is not within the renewal
   period, it should do nothing.

3) letsencrypt-install-txt-record

   This role runs on the adns server.  It installs the TXT records
   generated in step 2 to the acme.opendev.org domain and then
   refreshes the server.  Hosts wanting certificates will have
   pre-provisioned CNAME records for _acme-challenge.host.opendev.org
   pointing to acme.opendev.org.

4) letsencrypt-create-certs

   This role runs on each host, reading the same variable as in step
   2.  However this time the acme.sh tool is run to authenticate and
   create the certificates, which should now work correctly via the
   TXT records from step 3.  After this, the host will have the
   full certificate material.

Testing is added via testinfra.  For testing purposes requests are
made to the staging letsencrypt servers and a self-signed certificate
is provisioned in step 4 (as the authentication is not available
during CI).  We test that the DNS TXT records are created locally on
the CI adns server, however.

Related-Spec: https://review.openstack.org/587283

Change-Id: I1f66da614751a29cc565b37cdc9ff34d70fdfd3f
2019-04-02 15:31:41 +11:00

720 lines
22 KiB
YAML

- job:
name: puppet-beaker-rspec-infra-system-config
parent: puppet-beaker-rspec-infra
vars:
project_src_dir: "{{ zuul.project.src_dir }}/modules/openstack_project"
- job:
name: puppet-beaker-rspec-infra-centos-7-system-config
parent: puppet-beaker-rspec-centos-7-infra
vars:
project_src_dir: "{{ zuul.project.src_dir }}/modules/openstack_project"
- job:
name: puppet-beaker-rspec-puppet-4-infra-system-config
parent: puppet-beaker-rspec-puppet-4-infra
vars:
project_src_dir: "{{ zuul.project.src_dir }}/modules/openstack_project"
- job:
name: puppet-beaker-rspec-puppet-4-centos-7-infra-system-config
parent: puppet-beaker-rspec-puppet-4-infra
vars:
project_src_dir: "{{ zuul.project.src_dir }}/modules/openstack_project"
# Image building jobs
- secret:
name: system-config-dockerhub
data:
username: opendevzuul
password: !encrypted/pkcs1-oaep
- FF2mwsdYo4b9QI/kEmeFdZS32I+OJUac2xy4JEQJaIevRK9E3rQ/11cBn7xX6zelA/htY
hENMM1gGTYO8BKgTVNijFfCiVCTYFJ9efsOrSeMdj2aLcq/OkZFeq8gMHAZDv+3qaiggq
hbDmldgSouNahhJByWMsDjYqnItfR8w1dYisNQBkDEJdNW5+Cin5WgHRYv2skui9pjumU
vWHjwLbITLEjb0hwVErggAzvED4x0UjNEMfb6KCpoXeKdlCaI8h1DXVD9RdQ4s1WvUCro
xwl4AIUyCmdtXbqW787PQ4j8BgdOi8RD40ZTSxpoV/bwT+/NrP3Jj2KVYdEKQJimzXnwL
goDclapu9aFrAaqwWSnWCRc7GQptSZ7DBXmA7MsNinBclWHX/chX5k9Q1+XifAhmKJZqP
ovkzEGtmUjTQZFjvuCtFxIWZEGVtGHDmgEkzevgVCeuHDTeIDlkAvA3+TT+/A70zyYdbP
EHOTSSjCU6RSlJxAXjNb5pzKjpeT7PyX2f4/8Zn3JnwTlfPJ0C7qdEYjQ54hbsKynjrpQ
7f6NN9pAeQS1pid5J7oH4f7YaKM78pnhoTxRl3LmeitIlAG+zCpiT4J+se2eG27ep4aUA
bYuO0L+KBosY6DcvEbR9TqkYLPgJuEPi/6SQr9bBOldEspXXAv9msyCB4VMhVU=
- job:
name: system-config-build-image
parent: opendev-build-docker-image
dependencies: opendev-buildset-registry
abstract: true
- job:
name: system-config-upload-image
parent: opendev-upload-docker-image
dependencies: opendev-buildset-registry
abstract: true
secrets:
name: docker_credentials
secret: system-config-dockerhub
pass-to-parent: true
- job:
name: system-config-promote-image
parent: opendev-promote-docker-image
abstract: true
secrets:
name: docker_credentials
secret: system-config-dockerhub
pass-to-parent: true
# Jinja-init jobs
- job:
name: system-config-build-image-jinja-init
description: Build a jinja-init image.
parent: system-config-build-image
vars: &jinja-init_vars
docker_images:
- context: docker/jinja-init
target: jinja-init
repository: opendevorg/jinja-init
files: &jinja-init_files
- docker/jinja-init/.*
- job:
name: system-config-upload-image-jinja-init
description: Build and upload a jinja-init image.
parent: system-config-upload-image
vars: *jinja-init_vars
files: *jinja-init_files
- job:
name: system-config-promote-image-jinja-init
description: Promote a previously published jinja-init image to latest.
parent: system-config-promote-image
vars: *jinja-init_vars
files: *jinja-init_files
# Gitea-init jobs
- job:
name: system-config-build-image-gitea-init
description: Build a gitea-init image.
parent: system-config-build-image
vars: &gitea-init_vars
docker_images:
- context: docker/gitea-init
target: gitea-init
repository: opendevorg/gitea-init
files: &gitea-init_files
- docker/gitea-init/.*
- job:
name: system-config-upload-image-gitea-init
description: Build and upload a gitea-init image.
parent: system-config-upload-image
vars: *gitea-init_vars
files: *gitea-init_files
- job:
name: system-config-promote-image-gitea-init
description: Promote a previously published gitea-init image to latest.
parent: system-config-promote-image
vars: *gitea-init_vars
files: *gitea-init_files
# Gitea jobs
- job:
name: system-config-build-image-gitea
description: Build a gitea image.
parent: system-config-build-image
vars: &gitea_vars
docker_images:
- context: docker/gitea
target: gitea
repository: opendevorg/gitea
- context: docker/gitea
target: gitea-openssh
repository: opendevorg/gitea-openssh
# Duplicate in the run-gitea job
files: &gitea_files
- docker/gitea/.*
- job:
name: system-config-upload-image-gitea
description: Build and upload a gitea image.
parent: system-config-upload-image
vars: *gitea_vars
files: *gitea_files
- job:
name: system-config-promote-image-gitea
description: Promote a previously published gitea image to latest.
parent: system-config-promote-image
vars: *gitea_vars
files: *gitea_files
# Gerrit jobs
- job:
name: system-config-build-image-gerrit
description: Build a gerrit image.
parent: system-config-build-image
pre-run: playbooks/zuul/gerrit/repos.yaml
required-projects: &gerrit_projects
- name: gerrit.googlesource.com/gerrit
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/commit-message-length-validator
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/download-commands
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/hooks
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/its-base
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/its-storyboard
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/javamelody
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/replication
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/reviewnotes
override-checkout: stable-2.15
- name: gerrit.googlesource.com/plugins/singleusergroup
override-checkout: stable-2.15
vars: &gerrit_vars
docker_images:
- context: docker/gerrit
target: gerrit
repository: opendevorg/gerrit
path: /home/zuul/src/gerrit.googlesource.com/gerrit
tags:
- 2.15
build_args:
- BAZEL_OPTS="--local_resources=4096,2.0,1.0"
files: &gerrit_files
- docker/gerrit/.*
- playbooks/zuul/gerrit/.*
- job:
name: system-config-upload-image-gerrit
description: Build and upload a gerrit image.
parent: system-config-upload-image
pre-run: playbooks/zuul/gerrit/repos.yaml
required-projects: *gerrit_projects
vars: *gerrit_vars
files: *gerrit_files
- job:
name: system-config-promote-image-gerrit
description: Promote a previously published gerrit image to latest.
parent: system-config-promote-image
vars: *gerrit_vars
files: *gerrit_files
# python-builder jobs
- job:
name: system-config-build-image-python-builder
description: Build a python-builder image.
parent: system-config-build-image
vars: &python-builder_vars
docker_images:
- context: docker/python-builder
repository: opendevorg/python-builder
files: &python-builder_files
- docker/python-builder/.*
- job:
name: system-config-upload-image-python-builder
description: Build and upload a python-builder image.
parent: system-config-upload-image
vars: *python-builder_vars
files: *python-builder_files
- job:
name: system-config-promote-image-python-builder
description: Promote a previously published python-builder image to latest.
parent: system-config-promote-image
vars: *python-builder_vars
files: *python-builder_files
# python-base jobs
- job:
name: system-config-build-image-python-base
description: Build a python-base image.
parent: system-config-build-image
vars: &python-base_vars
docker_images:
- context: docker/python-base
repository: opendevorg/python-base
files: &python-base_files
- docker/python-base/.*
- job:
name: system-config-upload-image-python-base
description: Build and upload a python-base image.
parent: system-config-upload-image
vars: *python-base_vars
files: *python-base_files
- job:
name: system-config-promote-image-python-base
description: Promote a previously published python-base image to latest.
parent: system-config-promote-image
vars: *python-base_vars
files: *python-base_files
# Role integration jobs. These test the top-level generic roles/*
# under Zuul. The range of platforms should be the same as those for
# openstack-zuul-jobs.
#
# NOTE(ianw): 2018-08 have left off Fedora & OpenSUSE because
# no roles currently run on them.
- job:
name: system-config-zuul-role-integration
description: |
Test roles provided by system-config with Zuul
abstract: true
parent: base
run: roles-test/base.yaml
files:
- roles/.*
- job:
name: system-config-zuul-role-integration-centos-7
parent: system-config-zuul-role-integration
nodeset:
nodes:
- name: base
label: centos-7
- name: puppet4
label: centos-7
groups:
- name: puppet3
nodes:
- base
- job:
name: system-config-zuul-role-integration-trusty
parent: system-config-zuul-role-integration
nodeset:
nodes:
- name: base
label: ubuntu-trusty
- name: puppet4
label: ubuntu-trusty
groups:
- name: puppet3
nodes:
- base
- job:
name: system-config-zuul-role-integration-xenial
parent: system-config-zuul-role-integration
nodeset:
nodes:
- name: base
label: ubuntu-xenial
- name: puppet4
label: ubuntu-xenial
groups:
- name: puppet3
nodes:
- base
- job:
name: system-config-zuul-role-integration-bionic
parent: system-config-zuul-role-integration
nodeset:
nodes:
- name: base
label: ubuntu-bionic
groups:
- name: puppet5
nodes:
- base
- job:
name: system-config-zuul-role-integration-debian-stable
parent: system-config-zuul-role-integration
nodeset:
nodes:
- name: base
label: debian-stretch
- project-template:
name: system-config-zuul-role-integration
check:
jobs:
- system-config-zuul-role-integration-centos-7
- system-config-zuul-role-integration-trusty
- system-config-zuul-role-integration-xenial
- system-config-zuul-role-integration-bionic
- system-config-zuul-role-integration-debian-stable
gate:
jobs:
- system-config-zuul-role-integration-centos-7
- system-config-zuul-role-integration-trusty
- system-config-zuul-role-integration-xenial
- system-config-zuul-role-integration-bionic
- system-config-zuul-role-integration-debian-stable
- job:
name: system-config-run
description: |
Run the "base" playbook for system-config hosts.
This is a parent job designed to be inherited.
abstract: true
pre-run: playbooks/zuul/run-base-pre.yaml
run: playbooks/zuul/run-base.yaml
post-run: playbooks/zuul/run-base-post.yaml
vars:
install_ansible_ara_enable: true
zuul_copy_output: "{{ copy_output | combine(host_copy_output | default({})) }}"
stage_dir: "{{ ansible_user_dir }}/zuul-output"
copy_output:
'/var/log/syslog': logs_txt
'/var/log/messages': logs_txt
'/var/log/docker': logs
host-vars:
bridge.openstack.org:
host_copy_output:
'{{ zuul.project.src_dir }}/junit.xml': logs
- job:
name: system-config-run-base
parent: system-config-run
description: |
Run the "base" playbook on each of the node types
currently in use.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: trusty
label: ubuntu-trusty
- name: xenial
label: ubuntu-xenial
- name: bionic
label: ubuntu-bionic
- name: centos7
label: centos-7
host-vars:
trusty:
ansible_python_interpreter: python2
centos7:
ansible_python_interpreter: python2
files:
- .zuul.yaml
- playbooks/.*
- roles/.*
- testinfra/.*
- job:
name: system-config-run-base-ansible-devel
parent: system-config-run-base
description: |
Run the base playbook with the latest ansible
required-projects:
- name: github.com/ansible/ansible
override-checkout: devel
- name: openstack/openstacksdk
- name: openstack/ara
vars:
bridge_ansible_name: '{{ ansible_user_dir}}/src/github.com/ansible/ansible'
bridge_ansible_version: null
bridge_openstacksdk_name: '{{ ansible_user_dir }}/src/git.openstack.org/openstack/openstacksdk'
bridge_openstacksdk_version: null
bridge_ara_name: '{{ ansible_user_dir}}/src/git.openstack.org/openstack/ara'
bridge_ara_version: null
- job:
name: system-config-run-eavesdrop
parent: system-config-run
description: |
Run the playbook for an eavesdrop server.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: eavesdrop01.openstack.org
label: ubuntu-xenial
files:
- .zuul.yaml
- playbooks/group_vars/eavesdrop.yaml
- testinfra/test_eavesdrop.py
- job:
name: system-config-run-letsencrypt
parent: system-config-run
description: |
Run the playbook for letsencrypt key acquisition
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns-letsencrypt.opendev.org
label: ubuntu-bionic
- name: letsencrypt01.opendev.org
label: ubuntu-bionic
- name: letsencrypt02.opendev.org
label: ubuntu-bionic
host-vars:
letsencrypt01.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
letsencrypt02.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
files:
- .zuul.yaml
- playbooks/group_vars/letsencrypt.yaml
- playbooks/roles/letsencrypt.*
- job:
name: system-config-run-nodepool
parent: system-config-run
description: |
Run the playbook for nodepool.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: nl01.openstack.org
label: ubuntu-xenial
- name: nb01.openstack.org
label: ubuntu-xenial
files:
- .zuul.yaml
- playbooks/group_vars/nodepool.yaml
- playbooks/group_vars/nodepool-builder.yaml
- playbooks/group_vars/nodepool-launcher.yaml
- playbooks/roles/configure-openstacksdk/
- playbooks/templates/clouds/
- testinfra/test_nodepool.py
- job:
name: system-config-run-docker
parent: system-config-run
description: |
Test docker installation and setup
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: bionic-docker
label: ubuntu-bionic
files:
- .zuul.yaml
- playbooks/roles/install-docker
- testinfra/test_docker.py
- job:
name: system-config-run-dns
parent: system-config-run
description: |
Run the playbook for dns.
required-projects:
- openstack-infra/zone-opendev.org
- openstack-infra/zone-zuul-ci.org
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns1.opendev.org
label: ubuntu-bionic
- name: ns1.opendev.org
label: ubuntu-bionic
host-vars:
adns1.opendev.org:
host_copy_output:
'/etc/bind/named.conf': logs
'/var/lib/bind/zones': logs
files:
- .zuul.yaml
- playbooks/group_vars/adns.yaml
- playbooks/group_vars/dns.yaml
- ^playbooks/host_vars/(ad)?ns\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/adns.yaml.j2
- playbooks/zuul/templates/group_vars/ns.yaml.j2
- playbooks/roles/master-nameserver/
- playbooks/roles/nameserver/
- testinfra/test_adns.py
- testinfra/test_ns.py
- job:
name: system-config-run-docker-registry
parent: system-config-run
description: |
Run the playbook for the docker registry.
dependencies: opendev-buildset-registry
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: insecure-ci-registry01.opendev.org
label: ubuntu-bionic
host-vars:
insecure-ci-registry01.opendev.org:
host_copy_output:
'/var/registry/auth': logs
'/var/registry/certs': logs
files:
- .zuul.yaml
- playbooks/group_vars/registry.yaml
- ^playbooks/host_vars/insecure-ci-registry\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/registry.yaml.j2
- playbooks/roles/registry/
- testinfra/test_registry.py
- job:
name: system-config-run-gitea
parent: system-config-run
description: |
Run the playbook for the gitea servers.
dependencies: opendev-buildset-registry
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: gitea-lb01.opendev.org
label: ubuntu-bionic
- name: gitea01.opendev.org
label: ubuntu-bionic
vars:
run_base_test_playbook: playbooks/zuul/test-gitea.yaml
host-vars:
gitea01.opendev.org:
host_copy_output:
'/var/gitea/conf': logs
'/var/gitea/certs': logs
'/var/gitea/logs': logs
gitea-lb01.opendev.org:
host_copy_output:
'/var/haproxy/etc': logs
files:
- .zuul.yaml
- playbooks/group_vars/gitea.yaml
- playbooks/group_vars/gitea-lb.yaml
- playbooks/zuul/templates/group_vars/gitea.yaml.j2
- playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
- playbooks/roles/gitea/
- playbooks/roles/gitea-git-repos/
- playbooks/roles/haproxy/
- testinfra/test_gitea.py
- testinfra/test_gitea_lb.py
# From gitea_files -- If we rebuild the image, we want to run
# this job as well.
- docker/gitea/.*
- job:
name: system-config-run-zuul-preview
parent: system-config-run
description: |
Run the playbook for the docker registry.
dependencies: opendev-buildset-registry
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zp01.opendev.org
label: ubuntu-bionic
files:
- .zuul.yaml
- playbooks/roles/zuul-preview/
- testinfra/test_zuul_preview.py
- job:
name: infra-prod-playbook
description: |
Run specified playbook against productions hosts.
This is a parent job designed to be inherited to enabled
CD deployment of our infrastructure. Set playbook_name to
specify the playbook relative to
bridge.openstack.org:/opt/system-config/playbooks
abstract: true
run: playbooks/zuul/run-production-playbook.yaml
vars:
ansible_forks: 5
nodeset:
nodes: []
- job:
name: infra-prod-zuul_reconfigure
parent: infra-prod-playbook
description: |
Perform a full Zuul reconfiguration.
This reloads the tenant config file and clears all configuration
caches.
vars:
playbook_name: zuul_reconfigure.yaml
- project:
templates:
- system-config-zuul-role-integration
- infra-puppet-apply-jobs
- infra-logstash-filter-jobs
- publish-tox-docs-infra
check:
jobs:
- opendev-buildset-registry
- tox-linters
- legacy-system-config-puppet-syntax-3
- puppet-beaker-rspec-infra-system-config
- puppet-beaker-rspec-infra-centos-7-system-config
- puppet-beaker-rspec-puppet-4-infra-system-config
- puppet-beaker-rspec-puppet-4-centos-7-infra-system-config
- system-config-run-base
- system-config-run-base-ansible-devel:
voting: false
- system-config-run-dns
- system-config-run-eavesdrop
- system-config-run-nodepool
- system-config-run-docker
- system-config-run-docker-registry
- system-config-run-gitea:
dependencies:
- name: system-config-build-image-gitea
soft: true
- system-config-run-zuul-preview
- system-config-run-letsencrypt
- system-config-build-image-jinja-init
- system-config-build-image-gitea-init
- system-config-build-image-gitea
- system-config-build-image-gerrit
- system-config-build-image-python-base
- system-config-build-image-python-builder
gate:
jobs:
- opendev-buildset-registry
- tox-linters
- legacy-system-config-puppet-syntax-3
- puppet-beaker-rspec-infra-system-config
- puppet-beaker-rspec-infra-centos-7-system-config
- puppet-beaker-rspec-puppet-4-infra-system-config
- puppet-beaker-rspec-puppet-4-centos-7-infra-system-config
- system-config-run-base
- system-config-run-dns
- system-config-run-eavesdrop
- system-config-run-nodepool
- system-config-run-docker
- system-config-run-docker-registry
- system-config-run-gitea:
dependencies:
- name: system-config-upload-image-gitea
soft: true
- system-config-run-zuul-preview
- system-config-run-letsencrypt
- system-config-upload-image-jinja-init
- system-config-upload-image-gitea-init
- system-config-upload-image-gitea
- system-config-upload-image-gerrit
- system-config-upload-image-python-base
- system-config-upload-image-python-builder
promote:
jobs:
- system-config-promote-image-jinja-init
- system-config-promote-image-gitea-init
- system-config-promote-image-gitea
- system-config-promote-image-gerrit
- system-config-promote-image-python-base
- system-config-promote-image-python-builder