Test Ceph upgrade in CI

Add CI jobs for testing an upgrade of a multinode system with Ceph
enabled. As for the existing upgrade job, we upgrade from the previous
release to the current release.

Change-Id: I931772ca4c63757769467a57c80dc0726a11167a
Depends-On: https://review.opendev.org/658163
This commit is contained in:
Mark Goddard 2019-05-09 16:12:10 +01:00
parent 86e9b0831c
commit 78ee0287a7
6 changed files with 81 additions and 16 deletions

View File

@ -2,13 +2,14 @@
- hosts: all - hosts: all
vars: vars:
kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible" kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
is_ceph: "{{ 'ceph' in scenario }}"
tasks: tasks:
- name: Prepare disks for Ceph or LVM - name: Prepare disks for Ceph or LVM
script: "setup_disks.sh {{ disk_type }}" script: "setup_disks.sh {{ disk_type }}"
when: scenario in ["ceph", "cinder-lvm"] when: scenario == "cinder-lvm" or is_ceph
become: true become: true
vars: vars:
disk_type: "{{ ceph_storetype if scenario == 'ceph' else 'cinder-lvm' }}" disk_type: "{{ ceph_storetype if is_ceph else 'cinder-lvm' }}"
ceph_storetype: "{{ hostvars[inventory_hostname].get('ceph_osd_storetype') }}" ceph_storetype: "{{ hostvars[inventory_hostname].get('ceph_osd_storetype') }}"
- hosts: primary - hosts: primary
@ -19,6 +20,8 @@
kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible" kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
need_build_image: false need_build_image: false
build_image_tag: "change_{{ zuul.change | default('none') }}" build_image_tag: "change_{{ zuul.change | default('none') }}"
is_upgrade: "{{ 'upgrade' in scenario }}"
is_ceph: "{{ 'ceph' in scenario }}"
tasks: tasks:
# FIXME: in multi node env, api_interface may be different on each node. # FIXME: in multi node env, api_interface may be different on each node.
- name: detect api_interface_name variable - name: detect api_interface_name variable
@ -44,10 +47,13 @@
# NOTE(mgoddard): This only affects the remote copy of the repo, not the # NOTE(mgoddard): This only affects the remote copy of the repo, not the
# one on the executor. # one on the executor.
- name: checkout the previous kolla-ansible branch - name: checkout the previous kolla-ansible branch
command: shell:
cmd: "git checkout stable/{{ previous_release | lower }}" cmd: |
git checkout stable/{{ previous_release | lower }}
echo "kolla-ansible checked out to:"
git log --pretty=oneline -1
chdir: "{{ kolla_ansible_src_dir }}" chdir: "{{ kolla_ansible_src_dir }}"
when: scenario == "upgrade" when: is_upgrade
- name: ensure /etc/kolla exists - name: ensure /etc/kolla exists
file: file:
@ -63,7 +69,7 @@
include_role: include_role:
role: remote-template role: remote-template
vars: vars:
is_previous_release: "{{ scenario == 'upgrade' }}" is_previous_release: "{{ is_upgrade }}"
infra_dockerhub_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082/" infra_dockerhub_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082/"
# Role variables. # Role variables.
remote_template_src: "{{ kolla_ansible_src_dir }}/{{ item.src }}" remote_template_src: "{{ kolla_ansible_src_dir }}/{{ item.src }}"
@ -87,7 +93,7 @@
# ceph.conf # ceph.conf
- src: "tests/templates/ceph-overrides.j2" - src: "tests/templates/ceph-overrides.j2"
dest: /etc/kolla/config/ceph.conf dest: /etc/kolla/config/ceph.conf
when: scenario == "ceph" when: "{{ 'ceph' in scenario }}"
# bifrost/dib.yml # bifrost/dib.yml
- src: "tests/templates/bifrost-dib-overrides.j2" - src: "tests/templates/bifrost-dib-overrides.j2"
dest: /etc/kolla/config/bifrost/dib.yml dest: /etc/kolla/config/bifrost/dib.yml
@ -183,7 +189,7 @@
chdir: "{{ kolla_ansible_src_dir }}" chdir: "{{ kolla_ansible_src_dir }}"
environment: environment:
ACTION: "{{ scenario }}" ACTION: "{{ scenario }}"
when: scenario != "upgrade" when: not is_upgrade
when: scenario != "bifrost" when: scenario != "bifrost"
# Upgrade: update config. # Upgrade: update config.
@ -197,8 +203,11 @@
# NOTE(mgoddard): This only affects the remote copy of the repo, not the # NOTE(mgoddard): This only affects the remote copy of the repo, not the
# one on the executor. # one on the executor.
- name: checkout the current kolla-ansible branch - name: checkout the current kolla-ansible branch
command: shell:
cmd: "git checkout {{ zuul.branch }}" cmd: |
git checkout {{ zuul.branch }}
echo "kolla-ansible checked out to:"
git log --pretty=oneline -1
chdir: "{{ kolla_ansible_src_dir }}" chdir: "{{ kolla_ansible_src_dir }}"
# Use the new kolla-ansible repo to generate config files. # Use the new kolla-ansible repo to generate config files.
@ -223,7 +232,7 @@
# ceph.conf # ceph.conf
- src: "tests/templates/ceph-overrides.j2" - src: "tests/templates/ceph-overrides.j2"
dest: /etc/kolla/config/ceph.conf dest: /etc/kolla/config/ceph.conf
when: "'ceph' in scenario" when: "{{ is_ceph }}"
when: item.when | default(true) when: item.when | default(true)
- name: upgrade kolla-ansible requirements - name: upgrade kolla-ansible requirements
@ -268,7 +277,7 @@
chdir: "{{ kolla_ansible_src_dir }}" chdir: "{{ kolla_ansible_src_dir }}"
environment: environment:
ACTION: "{{ scenario }}" ACTION: "{{ scenario }}"
when: scenario == "upgrade" when: is_upgrade
# Bifrost testing. # Bifrost testing.
- block: - block:

View File

@ -36,7 +36,7 @@ openstack_release: "{{ previous_release }}"
{% endif %} {% endif %}
{% endif %} {% endif %}
{% if scenario == "ceph" %} {% if is_ceph %}
enable_ceph: "yes" enable_ceph: "yes"
enable_cinder: "yes" enable_cinder: "yes"
ceph_pool_pg_num: 8 ceph_pool_pg_num: 8

View File

@ -24,7 +24,7 @@ function test_openstack_logged {
fi fi
echo "SUCCESS: Server creation" echo "SUCCESS: Server creation"
if [[ $ACTION = "ceph" ]] || [[ $ACTION == "cinder-lvm" ]]; then if [[ $ACTION =~ "ceph" ]] || [[ $ACTION == "cinder-lvm" ]]; then
echo "TESTING: Cinder volume attachment" echo "TESTING: Cinder volume attachment"
openstack volume create --size 2 test_volume openstack volume create --size 2 test_volume
attempt=1 attempt=1
@ -51,6 +51,18 @@ function test_openstack_logged {
sleep 10 sleep 10
done done
openstack server remove volume kolla_boot_test test_volume openstack server remove volume kolla_boot_test test_volume
attempt=1
while [[ $(openstack volume show test_volume -f value -c status) != "available" ]]; do
echo "Volume not detached yet"
attempt=$((attempt+1))
if [[ $attempt -eq 10 ]]; then
echo "Volume failed to detach"
openstack volume show test_volume
return 1
fi
sleep 10
done
openstack volume delete test_volume
echo "SUCCESS: Cinder volume attachment" echo "SUCCESS: Cinder volume attachment"
fi fi
@ -85,7 +97,11 @@ function test_openstack_logged {
function test_openstack { function test_openstack {
echo "Testing OpenStack" echo "Testing OpenStack"
test_openstack_logged > /tmp/logs/ansible/test-openstack 2>&1 log_file=/tmp/logs/ansible/test-openstack
if [[ -f $log_file ]]; then
log_file=${log_file}-upgrade
fi
test_openstack_logged > $log_file 2>&1
result=$? result=$?
if [[ $result != 0 ]]; then if [[ $result != 0 ]]; then
echo "Testing OpenStack failed. See ansible/test-openstack for details" echo "Testing OpenStack failed. See ansible/test-openstack for details"

View File

@ -27,7 +27,7 @@ EOF
GATE_IMAGES="bifrost" GATE_IMAGES="bifrost"
fi fi
if [[ $ACTION == "ceph" ]]; then if [[ $ACTION =~ "ceph" ]]; then
GATE_IMAGES+=",ceph,cinder" GATE_IMAGES+=",ceph,cinder"
fi fi

View File

@ -141,6 +141,42 @@
base_distro: ubuntu base_distro: ubuntu
install_type: source install_type: source
- job:
name: kolla-ansible-ubuntu-source-upgrade-ceph
parent: kolla-ansible-upgrade-base
nodeset: kolla-ansible-bionic-multi
timeout: 9000
voting: false
vars:
base_distro: ubuntu
install_type: source
scenario: upgrade-ceph
host-vars:
primary:
ceph_osd_storetype: filestore
secondary1:
ceph_osd_storetype: bluestore
secondary2:
ceph_osd_storetype: bluestore
- job:
name: kolla-ansible-centos-source-upgrade-ceph
parent: kolla-ansible-upgrade-base
nodeset: kolla-ansible-centos-multi
timeout: 9000
voting: false
vars:
base_distro: centos
install_type: source
scenario: upgrade-ceph
host-vars:
primary:
ceph_osd_storetype: filestore
secondary1:
ceph_osd_storetype: bluestore
secondary2:
ceph_osd_storetype: bluestore
- job: - job:
name: kolla-ansible-bifrost-centos-source name: kolla-ansible-bifrost-centos-source
parent: kolla-ansible-bifrost-base parent: kolla-ansible-bifrost-base

View File

@ -29,6 +29,8 @@
files: ^ansible\/roles\/(barbican|heat|mistral|redis|tacker)\/.* files: ^ansible\/roles\/(barbican|heat|mistral|redis|tacker)\/.*
- kolla-ansible-centos-source-upgrade - kolla-ansible-centos-source-upgrade
- kolla-ansible-ubuntu-source-upgrade - kolla-ansible-ubuntu-source-upgrade
- kolla-ansible-centos-source-upgrade-ceph
- kolla-ansible-ubuntu-source-upgrade-ceph
- kolla-ansible-centos-binary - kolla-ansible-centos-binary
- kolla-ansible-ubuntu-binary - kolla-ansible-ubuntu-binary
gate: gate:
@ -46,3 +48,5 @@
- kolla-ansible-centos-source-cinder-lvm - kolla-ansible-centos-source-cinder-lvm
- kolla-ansible-centos-source-upgrade - kolla-ansible-centos-source-upgrade
- kolla-ansible-ubuntu-source-upgrade - kolla-ansible-ubuntu-source-upgrade
- kolla-ansible-centos-source-upgrade-ceph
- kolla-ansible-ubuntu-source-upgrade-ceph