Replace native zuul jobs with tox based jobs

It's becoming more difficult to support a deployment installer via
zuul-executor. There is some functionality that we want to use in
ansible, but doesn't yet exist from zuul-executors. This includes
groups of groups in inventory files, group_vars next to inventory
files, and newer versions of ansible.

While I'm confident zuul will be able to do this in the future, we can
switch to tox based jobs for now with the goal to migrate back.

Change-Id: Iaace91a631dfc9528a07024c680d51e6d5a3d52f
Signed-off-by: Paul Belanger <pabelanger@redhat.com>
This commit is contained in:
Paul Belanger 2018-04-09 20:14:26 -04:00
parent 0a1b5f5834
commit 501f7eb97d
No known key found for this signature in database
GPG Key ID: 611A80832067AF38
9 changed files with 80 additions and 367 deletions

View File

@ -1,17 +0,0 @@
- job:
name: windmill-tox-ubuntu-xenial
pre-run: tests/playbooks/pre.yaml
run: tests/playbooks/run.yaml
nodeset: ubuntu-xenial
required-projects:
- openstack/ansible-role-diskimage-builder
- openstack/ansible-role-gearman
- openstack/ansible-role-logrotate
- openstack/ansible-role-nodepool
- openstack/ansible-role-shade
- openstack/ansible-role-ssh
- openstack/ansible-role-statsd
- openstack/ansible-role-sudoers
- openstack/ansible-role-virtualenv
- openstack/ansible-role-zookeeper
- openstack/ansible-role-zuul

View File

@ -1,62 +1,25 @@
- job:
name: windmill-base-minimal
parent: base-minimal
pre-run: tests/base-minimal.yaml
roles:
- zuul: openstack-infra/zuul-jobs
- zuul: openstack-infra/openstack-zuul-jobs
timeout: 3600
- job:
name: windmill-base
parent: windmill-base-minimal
run: playbooks/site.yaml
post-run:
- playbooks/prove.yaml
- tests/collect-logs.yaml
roles:
- zuul: openstack/ansible-role-diskimage-builder
name: openstack.diskimage-builder
- zuul: openstack/ansible-role-gearman
name: openstack.gearman
- zuul: openstack/ansible-role-logrotate
name: openstack.logrotate
- zuul: openstack/ansible-role-nodepool
name: openstack.nodepool
- zuul: openstack/ansible-role-shade
name: openstack.shade
- zuul: openstack/ansible-role-statsd
name: openstack.statsd
- zuul: openstack/ansible-role-ssh
name: openstack.ssh
- zuul: openstack/ansible-role-sudoers
name: openstack.sudoers
- zuul: openstack/ansible-role-virtualenv
name: openstack.virtualenv
- zuul: openstack/ansible-role-zookeeper
name: openstack.zookeeper
- zuul: openstack/ansible-role-zuul
name: openstack.zuul
pre-run: tests/playbooks/pre.yaml
run: tests/playbooks/run.yaml
post-run: tests/playbooks/post.yaml
required-projects:
- openstack/ansible-role-diskimage-builder
- openstack/ansible-role-gearman
- openstack/ansible-role-logrotate
- openstack/ansible-role-nodepool
- openstack/ansible-role-shade
- openstack/ansible-role-ssh
- openstack/ansible-role-statsd
- openstack/ansible-role-sudoers
- openstack/ansible-role-virtualenv
- openstack/ansible-role-zookeeper
- openstack/ansible-role-zuul
- openstack/diskimage-builder
- openstack-infra/gear
- openstack-infra/nodepool
- openstack-infra/shade
- openstack-infra/zuul
vars:
# NOTE(pabelanger): Because we want to test Depends-On patches in the
# gate, we don't want our roles to update git after we pushed the repos
# from zuul-executors.
diskimage_builder_git_update: false
gear_git_update: false
nodepool_git_update: false
shade_git_update: false
zuul_git_update: false
# NOTE(pabelanger): In the gate, we want to create a zuul-test user /
# group as zuul already exists on our images.
zuul_user_name: zuul-test
zuul_user_group: zuul-test
zuul_user_home: /var/lib/zuul
- job:
name: windmill-centos-7
@ -66,149 +29,14 @@
- job:
name: windmill-fedora-27
parent: windmill-base
nodeset:
nodes:
- name:
- bastion
- gear01
- nb01
- nl01
- statsd01
- ze01
- zf01
- zk01
- zm01
- zs01
- zw01
label: fedora-27
groups:
- name: gear
nodes:
- gear01
- name: nodepool-builder
nodes:
- nb01
- name: nodepool-launcher
nodes:
- nl01
- name: statsd
nodes:
- statsd01
- name: zookeeper
nodes:
- zk01
- name: zuul-executor
nodes:
- ze01
- name: zuul-fingergw
nodes:
- zf01
- name: zuul-merger
nodes:
- zm01
- name: zuul-scheduler
nodes:
- zs01
- name: zuul-web
nodes:
- zw01
nodeset: fedora-27
- job:
name: windmill-ubuntu-bionic
parent: windmill-base
nodeset:
nodes:
- name:
- bastion
- gear01
- nb01
- nl01
- statsd01
- ze01
- zf01
- zk01
- zm01
- zs01
- zw01
label: ubuntu-bionic
groups:
- name: gear
nodes:
- gear01
- name: nodepool-builder
nodes:
- nb01
- name: nodepool-launcher
nodes:
- nl01
- name: statsd
nodes:
- statsd01
- name: zookeeper
nodes:
- zk01
- name: zuul-executor
nodes:
- ze01
- name: zuul-fingergw
nodes:
- zf01
- name: zuul-merger
nodes:
- zm01
- name: zuul-scheduler
nodes:
- zs01
- name: zuul-web
nodes:
- zw01
nodeset: ubuntu-bionic
- job:
name: windmill-ubuntu-xenial
parent: windmill-base
nodeset:
nodes:
- name:
- bastion
- gear01
- nb01
- nl01
- statsd01
- ze01
- zf01
- zk01
- zm01
- zs01
- zw01
label: ubuntu-xenial
groups:
- name: gear
nodes:
- gear01
- name: nodepool-builder
nodes:
- nb01
- name: nodepool-launcher
nodes:
- nl01
- name: statsd
nodes:
- statsd01
- name: zookeeper
nodes:
- zk01
- name: zuul-executor
nodes:
- ze01
- name: zuul-fingergw
nodes:
- zf01
- name: zuul-merger
nodes:
- zm01
- name: zuul-scheduler
nodes:
- zs01
- name: zuul-web
nodes:
- zw01
nodeset: ubuntu-xenial

View File

@ -4,9 +4,3 @@
- windmill-jobs-fedora-27
- windmill-jobs-bionic
- windmill-jobs-xenial
check:
jobs:
- windmill-tox-ubuntu-xenial
gate:
jobs:
- windmill-tox-ubuntu-xenial

View File

@ -19,4 +19,3 @@
- name: Setup test.prove role
include_role:
name: test.prove
when: zuul_success | bool

View File

@ -1,27 +0,0 @@
- hosts: bastion
roles:
- configure-unbound
- use-cached-repos
# - mirror-workspace-git-repos
- mirror-info
- role: configure-mirrors
set_apt_mirrors_trusted: True
tasks:
- name: Synchronize src repos to workspace directory.
synchronize:
dest: .
src: "{{ zuul.executor.src_root }}"
no_log: true
# NOTE(pabelanger): Because use-cache-repos force the delete of origin, the
# git module from ansible will not setup origin correctly. This is like a
# bug in ansible, and should be discussed upstream.
- name: Add origin to local git repos
command: "git remote add origin https://{{ item.canonical_name }}"
args:
chdir: "{{ ansible_user_dir }}/{{ item.src_dir}}"
with_items: "{{ zuul.projects.values() | list }}"
# ANSIBLE0006: git remote is not supported by ansible module
tags:
- skip_ansible_lint

View File

@ -1,46 +1,20 @@
- hosts: localhost
tasks:
- name: Ensure logs directory exists
file:
path: "{{ item }}"
state: directory
with_items:
- "{{ zuul.executor.log_root }}/logs"
- "{{ zuul.executor.log_root }}/logs/gear01/var/log"
- "{{ zuul.executor.log_root }}/logs/nb01/var/log"
- "{{ zuul.executor.log_root }}/logs/nl01/var/log"
- "{{ zuul.executor.log_root }}/logs/statsd01/var/log"
- "{{ zuul.executor.log_root }}/logs/ze01/var/log"
- "{{ zuul.executor.log_root }}/logs/zf01/var/log"
- "{{ zuul.executor.log_root }}/logs/zm01/var/log"
- "{{ zuul.executor.log_root }}/logs/zs01/var/log"
- "{{ zuul.executor.log_root }}/logs/zw01/var/log"
- hosts: statsd01
tasks:
- name: Ensure journald logs directory exists
file:
path: "~/logs/{{ inventory_hostname }}/journal"
path: "~/logs/{{ inventory_hostname }}/var/log/journal"
state: directory
- name: Collect journald logs
become: yes
shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
args:
creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
with_items:
- statsd
- name: Collect journald log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
mode: pull
src: "~/logs/{{ inventory_hostname }}/journal"
verify_host: true
- name: Collect statsd log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
dest: "~/logs/{{ inventory_hostname }}"
mode: pull
rsync_opts:
- "--relative"
@ -54,27 +28,19 @@
tasks:
- name: Ensure journald logs directory exists
file:
path: "~/logs/{{ inventory_hostname }}/journal"
path: "~/logs/{{ inventory_hostname }}/var/log/journal"
state: directory
- name: Collect journald logs
become: yes
shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
args:
creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
with_items:
- gear
- name: Collect journald log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
mode: pull
src: "~/logs/{{ inventory_hostname }}/journal"
verify_host: true
- name: Collect gear log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
dest: "~/logs/{{ inventory_hostname }}"
mode: pull
recursive: false
rsync_opts:
@ -89,27 +55,19 @@
tasks:
- name: Ensure journald logs directory exists
file:
path: "~/logs/{{ inventory_hostname }}/journal"
path: "~/logs/{{ inventory_hostname }}/var/log/journal"
state: directory
- name: Collect journald logs
become: yes
shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
args:
creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
with_items:
- nodepool-builder
- name: Collect journald log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
mode: pull
src: "~/logs/{{ inventory_hostname }}/journal"
verify_host: true
- name: Collect nodepool log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
dest: "~/logs/{{ inventory_hostname }}"
mode: pull
recursive: false
rsync_opts:
@ -127,27 +85,19 @@
tasks:
- name: Ensure journald logs directory exists
file:
path: "~/logs/{{ inventory_hostname }}/journal"
path: "~/logs/{{ inventory_hostname }}/var/log/journal"
state: directory
- name: Collect journald logs
become: yes
shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
args:
creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
with_items:
- nodepool-launcher
- name: Collect journald log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
mode: pull
src: "~/logs/{{ inventory_hostname }}/journal"
verify_host: true
- name: Collect nl01 log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
dest: "~/logs/{{ inventory_hostname }}"
mode: pull
recursive: false
rsync_opts:
@ -164,27 +114,19 @@
tasks:
- name: Ensure journald logs directory exists
file:
path: "~/logs/{{ inventory_hostname }}/journal"
path: "~/logs/{{ inventory_hostname }}/var/log/journal"
state: directory
- name: Collect journald logs
become: yes
shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
args:
creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
with_items:
- zuul-executor
- name: Collect journald log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
mode: pull
src: "~/logs/{{ inventory_hostname }}/journal"
verify_host: true
- name: Collect ze01 log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
dest: "~/logs/{{ inventory_hostname }}"
mode: pull
recursive: false
rsync_opts:
@ -202,27 +144,19 @@
tasks:
- name: Ensure journald logs directory exists
file:
path: "~/logs/{{ inventory_hostname }}/journal"
path: "~/logs/{{ inventory_hostname }}/var/log/journal"
state: directory
- name: Collect journald logs
become: yes
shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
args:
creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
with_items:
- zuul-fingergw
- name: Collect journald log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
mode: pull
src: "~/logs/{{ inventory_hostname }}/journal"
verify_host: true
- name: Collect zf01 log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname}}"
dest: "~/logs/{{ inventory_hostname}}"
mode: pull
recursive: false
rsync_opts:
@ -240,27 +174,19 @@
tasks:
- name: Ensure journald logs directory exists
file:
path: "~/logs/{{ inventory_hostname }}/journal"
path: "~/logs/{{ inventory_hostname }}/var/log/journal"
state: directory
- name: Collect journald logs
become: yes
shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
args:
creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
with_items:
- zuul-merger
- name: Collect journald log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
mode: pull
src: "~/logs/{{ inventory_hostname }}/journal"
verify_host: true
- name: Collect zm01 log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
dest: "~/logs/{{ inventory_hostname }}"
mode: pull
recursive: false
rsync_opts:
@ -278,27 +204,19 @@
tasks:
- name: Ensure journald logs directory exists
file:
path: "~/logs/{{ inventory_hostname }}/journal"
path: "~/logs/{{ inventory_hostname }}/var/log/journal"
state: directory
- name: Collect journald logs
become: yes
shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
args:
creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
with_items:
- zuul-scheduler
- name: Collect journald log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
mode: pull
src: "~/logs/{{ inventory_hostname }}/journal"
verify_host: true
- name: Collect zs01 log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
dest: "~/logs/{{ inventory_hostname }}"
mode: pull
recursive: false
rsync_opts:
@ -316,27 +234,19 @@
tasks:
- name: Ensure journald logs directory exists
file:
path: "~/logs/{{ inventory_hostname }}/journal"
path: "~/logs/{{ inventory_hostname }}/var/log/journal"
state: directory
- name: Collect journald logs
become: yes
shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
args:
creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
with_items:
- zuul-web
- name: Collect journald log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
mode: pull
src: "~/logs/{{ inventory_hostname }}/journal"
verify_host: true
- name: Collect zw01 log files
synchronize:
dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
dest: "~/logs/{{ inventory_hostname }}"
mode: pull
recursive: false
rsync_opts:

View File

@ -1,3 +1,11 @@
# NOTE(pabelanger): Because we want to test Depends-On patches in the
# gate, we don't want our roles to update git after we pushed the repos
# from zuul-executors.
diskimage_builder_git_update: false
gear_git_update: false
nodepool_git_update: false
shade_git_update: false
zuul_git_update: false
# NOTE(pabelanger): In the gate, we want to create a zuul-test user /
# group as zuul already exists on our images.
zuul_user_name: zuul-test

13
tests/playbooks/post.yaml Normal file
View File

@ -0,0 +1,13 @@
- hosts: all
tasks:
- name: Run ansible-playbook for collect-logs.yaml
args:
chdir: "{{ zuul.project.src_dir }}"
shell: tox -evenv -- ansible-playbook -i inventory/single-node tests/collect-logs.yaml
- name: Collect log files
synchronize:
dest: "{{ zuul.executor.log_root }}"
mode: pull
src: ~/logs
verify_host: true

View File

@ -5,7 +5,12 @@
chdir: "{{ zuul.project.src_dir }}"
shell: tox -evenv -- ansible-playbook -i inventory/single-node playbooks/bastion.yaml
- name: Run ansible-playbook
- name: Run ansible-playbook for site.yaml
args:
chdir: "{{ zuul.project.src_dir }}"
shell: tox -evenv -- ansible-playbook -i inventory/single-node playbooks/site.yaml --extra-vars "@tests/extra-vars.yaml"
- name: Run ansible-playbook for prove.yaml
args:
chdir: "{{ zuul.project.src_dir }}"
shell: tox -evenv -- ansible-playbook -i inventory/single-node playbooks/prove.yaml