Test Ceph upgrade in CI

Add CI jobs for testing an upgrade of a multinode system with Ceph
enabled. As for the existing upgrade job, we upgrade from the previous
release to the current release.

Change-Id: I931772ca4c63757769467a57c80dc0726a11167a
Depends-On: https://review.opendev.org/658162
(cherry picked from commit 78ee0287a7)

tests/run.yml

@@ -20,6 +20,7 @@
     kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
     need_build_image: false
     build_image_tag: "change_{{ zuul.change | default('none') }}"
+    is_upgrade: "{{ 'upgrade' in scenario }}"
     is_ceph: "{{ 'ceph' in scenario }}"
   tasks:
     # FIXME: in multi node env, api_interface may be different on each node.
@@ -46,10 +47,13 @@
     # NOTE(mgoddard): This only affects the remote copy of the repo, not the
     # one on the executor.
     - name: checkout the previous kolla-ansible branch
-      command:
-        cmd: "git checkout stable/{{ previous_release | lower }}"
+      shell:
+        cmd: |
+          git checkout stable/{{ previous_release | lower }}
+          echo "kolla-ansible checked out to:"
+          git log --pretty=oneline  -1
         chdir: "{{ kolla_ansible_src_dir }}"
-      when: scenario == "upgrade"
+      when: is_upgrade
 
     - name: ensure /etc/kolla exists
       file:
@@ -65,7 +69,7 @@
       include_role:
         role: remote-template
       vars:
-        is_previous_release: "{{ scenario == 'upgrade' }}"
+        is_previous_release: "{{ is_upgrade }}"
         infra_dockerhub_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082/"
         # Role variables.
         remote_template_src: "{{ kolla_ansible_src_dir }}/{{ item.src }}"
@@ -185,7 +189,7 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           environment:
             ACTION: "{{ scenario }}"
-          when: scenario != "upgrade"
+          when: not is_upgrade
       when: scenario != "bifrost"
 
     # Upgrade: update config.
@@ -199,8 +203,11 @@
         # NOTE(mgoddard): This only affects the remote copy of the repo, not the
         # one on the executor.
         - name: checkout the current kolla-ansible branch
-          command:
-            cmd: "git checkout {{ zuul.branch }}"
+          shell:
+            cmd: |
+              git checkout {{ zuul.branch }}
+              echo "kolla-ansible checked out to:"
+              git log --pretty=oneline  -1
             chdir: "{{ kolla_ansible_src_dir }}"
 
         # Use the new kolla-ansible repo to generate config files.
@@ -270,7 +277,7 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           environment:
             ACTION: "{{ scenario }}"
-      when: scenario == "upgrade"
+      when: is_upgrade
 
     # Bifrost testing.
     - block:

tests/templates/globals-default.j2

@@ -13,7 +13,7 @@ kolla_internal_vip_address: "{{ api_interface_address if hostvars | length > 2 e
 enable_haproxy: "{{ 'no' if hostvars | length > 2 else 'yes' }}"
 # TODO(mgoddard): Remove this in the Train cycle when heat is enabled in the
 # initial deployment.
-enable_heat: "{{ scenario != 'upgrade' or previous_release != 'rocky' }}"
+enable_heat: "{{ not is_upgrade or previous_release != 'rocky' }}"
 neutron_external_interface: "fake_interface"
 openstack_logging_debug: "True"
 openstack_service_workers: "1"

tests/test-openstack.sh

@@ -97,7 +97,11 @@ function test_openstack_logged {
 
 function test_openstack {
     echo "Testing OpenStack"
-    test_openstack_logged > /tmp/logs/ansible/test-openstack 2>&1
+    log_file=/tmp/logs/ansible/test-openstack
+    if [[ -f $log_file ]]; then
+        log_file=${log_file}-upgrade
+    fi
+    test_openstack_logged > $log_file 2>&1
     result=$?
     if [[ $result != 0 ]]; then
         echo "Testing OpenStack failed. See ansible/test-openstack for details"

zuul.d/jobs.yaml

@@ -141,6 +141,42 @@
       base_distro: ubuntu
       install_type: source
 
+- job:
+    name: kolla-ansible-ubuntu-source-upgrade-ceph
+    parent: kolla-ansible-upgrade-base
+    nodeset: kolla-ansible-bionic-multi
+    timeout: 9000
+    voting: false
+    vars:
+      base_distro: ubuntu
+      install_type: source
+      scenario: upgrade-ceph
+    host-vars:
+      primary:
+        ceph_osd_storetype: filestore
+      secondary1:
+        ceph_osd_storetype: bluestore
+      secondary2:
+        ceph_osd_storetype: bluestore
+
+- job:
+    name: kolla-ansible-centos-source-upgrade-ceph
+    parent: kolla-ansible-upgrade-base
+    nodeset: kolla-ansible-centos-multi
+    timeout: 9000
+    voting: false
+    vars:
+      base_distro: centos
+      install_type: source
+      scenario: upgrade-ceph
+    host-vars:
+      primary:
+        ceph_osd_storetype: filestore
+      secondary1:
+        ceph_osd_storetype: bluestore
+      secondary2:
+        ceph_osd_storetype: bluestore
+
 - job:
     name: kolla-ansible-bifrost-centos-source
     parent: kolla-ansible-bifrost-base

zuul.d/project.yaml

@@ -30,6 +30,8 @@
             files: ^ansible\/roles\/(barbican|heat|mistral|redis|tacker)\/.*
         - kolla-ansible-centos-source-upgrade
         - kolla-ansible-ubuntu-source-upgrade
+        - kolla-ansible-centos-source-upgrade-ceph
+        - kolla-ansible-ubuntu-source-upgrade-ceph
         - kolla-ansible-centos-binary
         - kolla-ansible-ubuntu-binary
     gate:
@@ -47,3 +49,5 @@
         - kolla-ansible-centos-source-cinder-lvm
         - kolla-ansible-centos-source-upgrade
         - kolla-ansible-ubuntu-source-upgrade
+        - kolla-ansible-centos-source-upgrade-ceph
+        - kolla-ansible-ubuntu-source-upgrade-ceph