Test Ceph upgrade in CI

Add CI jobs for testing an upgrade of a multinode system with Ceph
enabled. As for the existing upgrade job, we upgrade from the previous
release to the current release.

Change-Id: I931772ca4c63757769467a57c80dc0726a11167a
Depends-On: https://review.opendev.org/658163

tests/run.yml

@@ -2,13 +2,14 @@
 - hosts: all
   vars:
     kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
+    is_ceph: "{{ 'ceph' in scenario }}"
   tasks:
     - name: Prepare disks for Ceph or LVM
       script: "setup_disks.sh {{ disk_type }}"
-      when: scenario in ["ceph", "cinder-lvm"]
+      when: scenario == "cinder-lvm" or is_ceph
       become: true
       vars:
-        disk_type: "{{ ceph_storetype if scenario == 'ceph' else 'cinder-lvm' }}"
+        disk_type: "{{ ceph_storetype if is_ceph else 'cinder-lvm' }}"
         ceph_storetype: "{{ hostvars[inventory_hostname].get('ceph_osd_storetype') }}"
 
 - hosts: primary
@@ -19,6 +20,8 @@
     kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
     need_build_image: false
     build_image_tag: "change_{{ zuul.change | default('none') }}"
+    is_upgrade: "{{ 'upgrade' in scenario }}"
+    is_ceph: "{{ 'ceph' in scenario }}"
   tasks:
     # FIXME: in multi node env, api_interface may be different on each node.
     - name: detect api_interface_name variable
@@ -44,10 +47,13 @@
     # NOTE(mgoddard): This only affects the remote copy of the repo, not the
     # one on the executor.
     - name: checkout the previous kolla-ansible branch
-      command:
+      shell:
-        cmd: "git checkout stable/{{ previous_release | lower }}"
+        cmd: |
+          git checkout stable/{{ previous_release | lower }}
+          echo "kolla-ansible checked out to:"
+          git log --pretty=oneline  -1
         chdir: "{{ kolla_ansible_src_dir }}"
-      when: scenario == "upgrade"
+      when: is_upgrade
 
     - name: ensure /etc/kolla exists
       file:
@@ -63,7 +69,7 @@
       include_role:
         role: remote-template
       vars:
-        is_previous_release: "{{ scenario == 'upgrade' }}"
+        is_previous_release: "{{ is_upgrade }}"
         infra_dockerhub_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082/"
         # Role variables.
         remote_template_src: "{{ kolla_ansible_src_dir }}/{{ item.src }}"
@@ -87,7 +93,7 @@
         # ceph.conf
         - src: "tests/templates/ceph-overrides.j2"
           dest: /etc/kolla/config/ceph.conf
-          when: scenario == "ceph"
+          when: "{{ 'ceph' in scenario }}"
         # bifrost/dib.yml
         - src: "tests/templates/bifrost-dib-overrides.j2"
           dest: /etc/kolla/config/bifrost/dib.yml
@@ -183,7 +189,7 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           environment:
             ACTION: "{{ scenario }}"
-          when: scenario != "upgrade"
+          when: not is_upgrade
       when: scenario != "bifrost"
 
     # Upgrade: update config.
@@ -197,8 +203,11 @@
         # NOTE(mgoddard): This only affects the remote copy of the repo, not the
         # one on the executor.
         - name: checkout the current kolla-ansible branch
-          command:
+          shell:
-            cmd: "git checkout {{ zuul.branch }}"
+            cmd: |
+              git checkout {{ zuul.branch }}
+              echo "kolla-ansible checked out to:"
+              git log --pretty=oneline  -1
             chdir: "{{ kolla_ansible_src_dir }}"
 
         # Use the new kolla-ansible repo to generate config files.
@@ -223,7 +232,7 @@
             # ceph.conf
             - src: "tests/templates/ceph-overrides.j2"
               dest: /etc/kolla/config/ceph.conf
-              when: "'ceph' in scenario"
+              when: "{{ is_ceph }}"
           when: item.when | default(true)
 
         - name: upgrade kolla-ansible requirements
@@ -268,7 +277,7 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           environment:
             ACTION: "{{ scenario }}"
-      when: scenario == "upgrade"
+      when: is_upgrade
 
     # Bifrost testing.
     - block:

tests/templates/globals-default.j2

@@ -36,7 +36,7 @@ openstack_release: "{{ previous_release }}"
 {% endif %}
 {% endif %}
 
-{% if scenario == "ceph" %}
+{% if is_ceph %}
 enable_ceph: "yes"
 enable_cinder: "yes"
 ceph_pool_pg_num: 8

tests/test-openstack.sh

@@ -24,7 +24,7 @@ function test_openstack_logged {
     fi
     echo "SUCCESS: Server creation"
 
-    if [[ $ACTION = "ceph" ]] || [[ $ACTION == "cinder-lvm" ]]; then
+    if [[ $ACTION =~ "ceph" ]] || [[ $ACTION == "cinder-lvm" ]]; then
         echo "TESTING: Cinder volume attachment"
         openstack volume create --size 2 test_volume
         attempt=1
@@ -51,6 +51,18 @@ function test_openstack_logged {
             sleep 10
         done
         openstack server remove volume kolla_boot_test test_volume
+        attempt=1
+        while [[ $(openstack volume show test_volume -f value -c status) != "available" ]]; do
+            echo "Volume not detached yet"
+            attempt=$((attempt+1))
+            if [[ $attempt -eq 10 ]]; then
+                echo "Volume failed to detach"
+                openstack volume show test_volume
+                return 1
+            fi
+            sleep 10
+        done
+        openstack volume delete test_volume
         echo "SUCCESS: Cinder volume attachment"
     fi
 
@@ -85,7 +97,11 @@ function test_openstack_logged {
 
 function test_openstack {
     echo "Testing OpenStack"
-    test_openstack_logged > /tmp/logs/ansible/test-openstack 2>&1
+    log_file=/tmp/logs/ansible/test-openstack
+    if [[ -f $log_file ]]; then
+        log_file=${log_file}-upgrade
+    fi
+    test_openstack_logged > $log_file 2>&1
     result=$?
     if [[ $result != 0 ]]; then
         echo "Testing OpenStack failed. See ansible/test-openstack for details"

tools/setup_gate.sh

@@ -27,7 +27,7 @@ EOF
         GATE_IMAGES="bifrost"
     fi
 
-    if [[ $ACTION == "ceph" ]]; then
+    if [[ $ACTION =~ "ceph" ]]; then
         GATE_IMAGES+=",ceph,cinder"
     fi
 

zuul.d/jobs.yaml

@@ -141,6 +141,42 @@
       base_distro: ubuntu
       install_type: source
 
+- job:
+    name: kolla-ansible-ubuntu-source-upgrade-ceph
+    parent: kolla-ansible-upgrade-base
+    nodeset: kolla-ansible-bionic-multi
+    timeout: 9000
+    voting: false
+    vars:
+      base_distro: ubuntu
+      install_type: source
+      scenario: upgrade-ceph
+    host-vars:
+      primary:
+        ceph_osd_storetype: filestore
+      secondary1:
+        ceph_osd_storetype: bluestore
+      secondary2:
+        ceph_osd_storetype: bluestore
+
+- job:
+    name: kolla-ansible-centos-source-upgrade-ceph
+    parent: kolla-ansible-upgrade-base
+    nodeset: kolla-ansible-centos-multi
+    timeout: 9000
+    voting: false
+    vars:
+      base_distro: centos
+      install_type: source
+      scenario: upgrade-ceph
+    host-vars:
+      primary:
+        ceph_osd_storetype: filestore
+      secondary1:
+        ceph_osd_storetype: bluestore
+      secondary2:
+        ceph_osd_storetype: bluestore
+
 - job:
     name: kolla-ansible-bifrost-centos-source
     parent: kolla-ansible-bifrost-base

zuul.d/project.yaml

@@ -29,6 +29,8 @@
             files: ^ansible\/roles\/(barbican|heat|mistral|redis|tacker)\/.*
         - kolla-ansible-centos-source-upgrade
         - kolla-ansible-ubuntu-source-upgrade
+        - kolla-ansible-centos-source-upgrade-ceph
+        - kolla-ansible-ubuntu-source-upgrade-ceph
         - kolla-ansible-centos-binary
         - kolla-ansible-ubuntu-binary
     gate:
@@ -46,3 +48,5 @@
         - kolla-ansible-centos-source-cinder-lvm
         - kolla-ansible-centos-source-upgrade
         - kolla-ansible-ubuntu-source-upgrade
+        - kolla-ansible-centos-source-upgrade-ceph
+        - kolla-ansible-ubuntu-source-upgrade-ceph