openstack/kolla-ansible
Code Issues Proposed changes

Test upgrades in CI

Browse Source 
This patch adds two new jobs:

* kolla-ansible-centos-source-upgrade
* kolla-ansible-ubuntu-source-upgrade

These jobs first deploy a control plane using the previous release of
Kolla Ansible, then upgrade to the current release.

Because we can't change the branch of the git repository on the Zuul
executor, we change the branch of the kolla-ansible repository on the
primary node to the branch of the previous release, in this case
stable/rocky. A new remote-template role has been added that supports
generating templates using a remote template source, to generate config
files using the previous kolla-ansible branch.

If the change being tested depends on a kolla change for the current
branch, then we build images. Rather than using the current
kolla-ansible version to tag the images, we now tag them with
change_<gerrit change ID>. This is because the version of kolla-ansible
will change from the previous release to the current one as we upgrade
the system.

Finally, it should be noted that the 'previous_release' variable in the
Zuul config needs to be updated with each release, since this sets the
release of kolla-ansible that is installed initially.

Depends-On: https://review.openstack.org/645089/
Depends-On: https://review.openstack.org/644250/
Depends-On: https://review.openstack.org/645816/
Depends-On: https://review.openstack.org/645840/
Change-Id: If301e0affcd55360fefe3b105f023ae5c47b0853
This commit is contained in:
Mark Goddard 2018-11-08 17:18:53 +00:00
parent 192dcd1e1b
commit c23c9b2c25
8 changed files with 246 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tests/roles/remote-template/defaults/main.yml Normal file
View File

@ -0,0 +1,12 @@
---
# Path to template file.
remote_template_src:
# Path to destination.
remote_template_dest:
# Whether to use 'become'.
remote_template_become: false
# Path on localhost to store a copy of the template.
remote_template_temp_path: "{{ zuul.executor.work_root }}/{{ remote_template_src | basename }}"

22
tests/roles/remote-template/tasks/main.yml Normal file
View File

@ -0,0 +1,22 @@
---
# Ensure the parent directory exists.
- name: "ensure {{ remote_template_dest | dirname }} exists"
file:
path: "{{ remote_template_dest | dirname }}"
state: "directory"
mode: 0777
become: "{{ remote_template_become }}"
# Template sources must be on localhost, so first fetch the remote template
# file.
- name: "fetch remote template {{ remote_template_src | basename }}"
fetch:
src: "{{ remote_template_src }}"
dest: "{{ remote_template_temp_path }}"
flat: true
- name: "template {{ remote_template_src | basename }}"
template:
src: "{{ remote_template_temp_path }}"
dest: "{{ remote_template_dest }}"
become: "{{ remote_template_become }}"

269
tests/run.yml
View File

@ -1,11 +1,10 @@
---
- hosts: all
vars:
kolla_ansible_src_dir: "src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
kolla_ansible_full_src_dir: "{{ zuul.executor.work_root }}/{{ kolla_ansible_src_dir }}"
kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
tasks:
- name: Prepare disks for Ceph or LVM
script: "{{ kolla_ansible_full_src_dir }}/tests/setup_disks.sh {{ disk_type }}"
script: "setup_disks.sh {{ disk_type }}"
when: scenario in ["ceph", "cinder-lvm"]
become: true
vars:
@ -16,23 +15,10 @@
vars:
kolla_inventory_path: "/etc/kolla/inventory"
logs_dir: "/tmp/logs"
kolla_ansible_src_dir: "src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
kolla_ansible_full_src_dir: "{{ zuul.executor.work_root }}/{{ kolla_ansible_src_dir }}"
kolla_ansible_src_dir: "{{ ansible_env.PWD }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
need_build_image: false
tasks:
- name: ensure /etc/kolla exists
file:
path: "/etc/kolla"
state: "directory"
mode: 0777
become: true
- name: copy default ansible kolla-ansible inventory
template:
src: "{{ kolla_ansible_full_src_dir }}/tests/templates/inventory.j2"
dest: "{{ kolla_inventory_path }}"
delegate_to: "primary"
# FIXME: in multi node env, api_interface may be different on each node.
- name: detect api_interface_name variable
vars:
@ -45,77 +31,78 @@
- hostvars[inventory_hostname][ansible_interface_name]['ipv4'] is defined
- hostvars[inventory_hostname][ansible_interface_name]['ipv4']['address'] == api_interface_address
with_items: "{{ ansible_interfaces }}"
delegate_to: "primary"
- name: detect whether need build images
set_fact:
need_build_image: true
when:
- item.project.short_name == "kolla"
- item.branch == zuul.branch
with_items: "{{ zuul['items'] }}"
- name: generate global.yml file
template:
src: "{{ kolla_ansible_full_src_dir }}/tests/templates/globals-default.j2"
dest: /etc/kolla/globals.yml
delegate_to: "primary"
# NOTE(mgoddard): This only affects the remote copy of the repo, not the
# one on the executor.
- name: checkout the previous kolla-ansible branch
command:
cmd: "git checkout stable/{{ previous_release | lower }}"
chdir: "{{ kolla_ansible_src_dir }}"
when: scenario == "upgrade"
- name: ensure nova conf overrides dir exists
- name: ensure /etc/kolla exists
file:
path: "/etc/kolla/config/nova"
state: "directory"
mode: 0777
when: scenario != "bifrost"
become: true
delegate_to: "primary"
- name: generate nova config overrides
template:
src: "{{ kolla_ansible_full_src_dir }}/tests/templates/nova-compute-overrides.j2"
dest: /etc/kolla/config/nova/nova-compute.conf
when: scenario != "bifrost"
delegate_to: "primary"
- name: ensure bifrost conf overrides dir exists
file:
path: "/etc/kolla/config/bifrost"
state: "directory"
mode: 0777
when: scenario == "bifrost"
become: true
delegate_to: "primary"
- name: generate bifrost DIB config overrides
template:
src: "{{ kolla_ansible_full_src_dir }}/tests/templates/bifrost-dib-overrides.j2"
dest: /etc/kolla/config/bifrost/dib.yml
when: scenario == "bifrost"
delegate_to: "primary"
- name: ensure /etc/docker exists
file:
path: "/etc/docker"
path: "/etc/kolla"
state: "directory"
mode: 0777
become: true
- name: create deamon.json for nodepool cache
# Use the initial repo to generate config files. For upgrade jobs, this
# repo is only available on the remote node, so use the remote-template
# role.
- name: generate configuration files
include_role:
role: remote-template
vars:
is_previous_release: "{{ scenario == 'upgrade' }}"
infra_dockerhub_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082/"
template:
src: "{{ kolla_ansible_full_src_dir }}/tests/templates/docker_daemon.json.j2"
dest: "/etc/docker/daemon.json"
become: true
# Role variables.
remote_template_src: "{{ kolla_ansible_src_dir }}/{{ item.src }}"
remote_template_dest: "{{ item.dest }}"
remote_template_become: "{{ item.become | default(false) }}"
with_items:
# Docker daemon.json
- src: "tests/templates/docker_daemon.json.j2"
dest: "/etc/docker/daemon.json"
become: true
# Ansible inventory
- src: "tests/templates/inventory.j2"
dest: "{{ kolla_inventory_path }}"
# globals.yml
- src: "tests/templates/globals-default.j2"
dest: /etc/kolla/globals.yml
# nova-compute.conf
- src: "tests/templates/nova-compute-overrides.j2"
dest: /etc/kolla/config/nova/nova-compute.conf
when: scenario != "bifrost"
# ceph.conf
- src: "tests/templates/ceph-overrides.j2"
dest: /etc/kolla/config/ceph.conf
when: scenario == "ceph"
# bifrost/dib.yml
- src: "tests/templates/bifrost-dib-overrides.j2"
dest: /etc/kolla/config/bifrost/dib.yml
when: scenario == "bifrost"
when: item.when | default(true)
- name: install kolla-ansible requirements
pip:
requirements: "{{ ansible_env.HOME }}/{{ kolla_ansible_src_dir }}/requirements.txt"
requirements: "{{ kolla_ansible_src_dir }}/requirements.txt"
become: true
- name: copy passwords.yml file
copy:
src: "{{ kolla_ansible_full_src_dir }}/etc/kolla/passwords.yml"
src: "{{ kolla_ansible_src_dir }}/etc/kolla/passwords.yml"
dest: /etc/kolla/passwords.yml
remote_src: true
- name: generate passwords
shell: "{{ kolla_ansible_src_dir }}/tools/generate_passwords.py"
@ -141,30 +128,133 @@
- all
delegate_to: "{{ item }}"
- name: generate ceph config overrides
template:
src: "{{ kolla_ansible_full_src_dir }}/tests/templates/ceph-overrides.j2"
dest: /etc/kolla/config/ceph.conf
when: scenario == "ceph"
delegate_to: "primary"
# NOTE(mgoddard): We are using the script module here and later to ensure
# we use the local copy of these scripts, rather than the one on the remote
# host, which could be checked out to a previous release (in an upgrade
# job).
- name: Run setup_gate.sh script
shell:
cmd: tools/setup_gate.sh
script:
cmd: ../tools/setup_gate.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
environment:
BASE_DISTRO: "{{ base_distro }}"
INSTALL_TYPE: "{{ install_type }}"
NODEPOOL_TARBALLS_MIRROR: "http://{{ zuul_site_mirror_fqdn }}:8080/tarballs"
BUILD_IMAGE: "{{ need_build_image }}"
TAG: "change_{{ zuul.change }}"
KOLLA_SRC_DIR: "{{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla"
ACTION: "{{ scenario }}"
# At this point we have generated all necessary configuration, and are
# ready to deploy the control plane services. Control flow now depends on
# the scenario being exercised.
# Deploy control plane. For upgrade jobs this is the previous release.
- block:
- name: Run deploy.sh script
script:
cmd: deploy.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
environment:
ACTION: "{{ scenario }}"
- name: Run test-openstack.sh script
script:
cmd: test-openstack.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
environment:
ACTION: "{{ scenario }}"
when: scenario not in ['scenario_nfv']
- name: Run test-scenario-nfv.sh script
script:
cmd: test-scenario-nfv.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
when: scenario == "scenario_nfv"
- name: Run reconfigure.sh script
script:
cmd: reconfigure.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
environment:
ACTION: "{{ scenario }}"
when: scenario != "upgrade"
when: scenario != "bifrost"
# Upgrade: update config.
- block:
- name: Run check-failure.sh script
script:
cmd: check-failure.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
# NOTE(mgoddard): This only affects the remote copy of the repo, not the
# one on the executor.
- name: checkout the current kolla-ansible branch
command:
cmd: "git checkout {{ zuul.branch }}"
chdir: "{{ kolla_ansible_src_dir }}"
# Use the new kolla-ansible repo to generate config files.
# This is the branch checked out on the executor, so we can use
# template directly rather than the remote-template role.
- name: Generate configuration files
template:
src: "{{ kolla_ansible_local_src_dir }}/{{ item.src }}"
dest: "{{ item.dest }}"
vars:
is_previous_release: false
with_items:
# Ansible inventory
- src: "tests/templates/inventory.j2"
dest: "{{ kolla_inventory_path }}"
# globals.yml
- src: "tests/templates/globals-default.j2"
dest: /etc/kolla/globals.yml
# nova-compute.conf
- src: "tests/templates/nova-compute-overrides.j2"
dest: /etc/kolla/config/nova/nova-compute.conf
# ceph.conf
- src: "tests/templates/ceph-overrides.j2"
dest: /etc/kolla/config/ceph.conf
when: "'ceph' in scenario"
when: item.when | default(true)
- name: upgrade kolla-ansible requirements
pip:
requirements: "{{ kolla_ansible_src_dir }}/requirements.txt"
become: true
# Update passwords.yml to include any new passwords added in this
# release.
- name: move passwords.yml to passwords.yml.old
command: mv /etc/kolla/passwords.yml /etc/kolla/passwords.yml.old
- name: copy passwords.yml file
copy:
src: "{{ kolla_ansible_src_dir }}/etc/kolla/passwords.yml"
dest: /etc/kolla/passwords.yml
remote_src: true
- name: generate new passwords
shell: "{{ kolla_ansible_src_dir }}/tools/generate_passwords.py"
- name: merge old and new passwords
shell: >-
{{ kolla_ansible_src_dir }}/tools/merge_passwords.py
--old /etc/kolla/passwords.yml.old
--new /etc/kolla/passwords.yml
--final /etc/kolla/passwords.yml
# Perform an upgrade to the in-development code.
- name: Run upgrade.sh script
shell:
cmd: tests/deploy.sh
cmd: tests/upgrade.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
environment:
@ -177,32 +267,9 @@
chdir: "{{ kolla_ansible_src_dir }}"
environment:
ACTION: "{{ scenario }}"
when: scenario not in ['scenario_nfv']
- name: Run test-scenario-nfv.sh script
shell:
cmd: tests/test-scenario-nfv.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
when: scenario == "scenario_nfv"
- name: Run reconfigure.sh script
shell:
cmd: tests/reconfigure.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
environment:
ACTION: "{{ scenario }}"
- name: Run upgrade.sh script
shell:
cmd: tests/upgrade.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
environment:
ACTION: "{{ scenario }}"
when: scenario != "bifrost"
when: scenario == "upgrade"
# Bifrost testing.
- block:
- name: Run deploy-bifrost.sh script
shell:

15
tests/templates/globals-default.j2
View File

@ -11,19 +11,32 @@ keepalived_virtual_router_id: "{{ 250 | random(1) }}"
{% if enable_core_openstack | bool %}
kolla_internal_vip_address: "{{ api_interface_address if hostvars | length > 2 else '169.254.169.10' }}"
enable_haproxy: "{{ 'no' if hostvars | length > 2 else 'yes' }}"
# TODO(mgoddard): Remove this in the Train cycle when heat is enabled in the
# initial deployment.
enable_heat: "{{ scenario != 'upgrade' or previous_release != 'rocky' }}"
neutron_external_interface: "fake_interface"
openstack_logging_debug: "True"
openstack_service_workers: "1"
{% endif %}
{% if need_build_image %}
{% if need_build_image and not is_previous_release %}
# NOTE(Jeffrey4l): use different a docker namespace name in case it pull image from hub.docker.io when deplying
docker_namespace: "lokolla"
docker_registry: "{{ api_interface_address }}:4000"
openstack_release: "change_{{ zuul.change }}"
{% else %}
# use docker hub images
docker_namespace: "kolla"
{% if need_build_image and is_previous_release %}
# NOTE(mgoddard): Ensure that the insecure local registry is trusted, since it
# will be the source of images during the upgrade.
docker_custom_option: "--insecure-registry {{ api_interface_address }}:4000"
{% endif %}
{% if not is_previous_release %}
openstack_release: "{{ zuul.branch | basename }}"
{% else %}
openstack_release: "{{ previous_release }}"
{% endif %}
{% endif %}
{% if scenario == "ceph" %}

19
tools/setup_gate.sh
View File

@ -8,19 +8,6 @@ export PYTHONUNBUFFERED=1
GIT_PROJECT_DIR=$(mktemp -d)
function clone_repos {
cat > /tmp/clonemap <<EOF
clonemap:
- name: openstack/kolla
dest: ${GIT_PROJECT_DIR}/kolla
- name: openstack/requirements
dest: ${GIT_PROJECT_DIR}/requirements
EOF
/usr/zuul-env/bin/zuul-cloner -m /tmp/clonemap --workspace "$(pwd)" \
--cache-dir /opt/git git://git.openstack.org \
openstack/kolla openstack/requirements
}
function setup_config {
# Use Infra provided pypi.
# Wheel package mirror may be not compatible. So do not enable it.
@ -56,9 +43,6 @@ EOF
GATE_IMAGES+=",tacker,mistral,redis,barbican"
fi
# Use the kolla-ansible tag rather than the kolla tag, since this is what
# kolla-ansible will use by default.
TAG=$(python -c "import pbr.version; print(pbr.version.VersionInfo('kolla-ansible'))")
cat <<EOF | sudo tee /etc/kolla/kolla-build.conf
[DEFAULT]
include_header = /etc/kolla/header
@ -75,7 +59,7 @@ logs_dir = /tmp/logs/build
gate = ${GATE_IMAGES}
EOF
mkdir -p /tmp/logs/build
mkdir -p /tmp/logs/build
if [[ "${DISTRO}" == "Debian" ]]; then
# Infra does not sign their mirrors so we ignore gpg signing in the gate
@ -138,7 +122,6 @@ function prepare_images {
}
clone_repos
setup_ansible
setup_config
setup_node

7
zuul.d/base.yaml
View File

@ -22,6 +22,13 @@
roles:
- zuul: openstack-infra/zuul-jobs
- job:
name: kolla-ansible-upgrade-base
parent: kolla-ansible-base
vars:
previous_release: rocky
scenario: upgrade
- job:
name: kolla-ansible-bifrost-base
parent: kolla-ansible-base

16
zuul.d/jobs.yaml
View File

@ -122,6 +122,22 @@
install_type: source
scenario: cinder-lvm
- job:
name: kolla-ansible-centos-source-upgrade
parent: kolla-ansible-upgrade-base
nodeset: kolla-ansible-centos
vars:
base_distro: centos
install_type: source
- job:
name: kolla-ansible-ubuntu-source-upgrade
parent: kolla-ansible-upgrade-base
nodeset: kolla-ansible-bionic
vars:
base_distro: ubuntu
install_type: source
- job:
name: kolla-ansible-bifrost-centos-source
parent: kolla-ansible-bifrost-base

6
zuul.d/project.yaml
View File

@ -28,10 +28,14 @@
files: ^ansible\/roles\/(zun|kuryr)\/.*
- kolla-ansible-centos-source-scenario-nfv:
files: ^ansible\/roles\/(barbican|heat|mistral|redis|tacker)\/.*
- kolla-ansible-centos-source-upgrade
- kolla-ansible-ubuntu-source-upgrade
gate:
jobs:
- kolla-ansible-centos-source
- kolla-ansible-ubuntu-source
- kolla-ansible-centos-source-upgrade
- kolla-ansible-ubuntu-source-upgrade
periodic:
jobs:
- kolla-ansible-bifrost-centos-source
@ -39,3 +43,5 @@
- kolla-ansible-centos-source-scenario-nfv
- kolla-ansible-ubuntu-source-cinder-lvm
- kolla-ansible-centos-source-cinder-lvm
- kolla-ansible-centos-source-upgrade
- kolla-ansible-ubuntu-source-upgrade