Ubuntu: add upgrade jobs in CI

Now that the previous release is Wallaby, we can add overcloud and seed
upgrades on Ubuntu.

Change-Id: I69868dca6e1f3e4a1bf123c9e7f1f3412162b264

playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2

@@ -10,6 +10,11 @@ docker_registry_mirrors:
 kolla_docker_namespace: "openstack.kolla"
 # use the published images from a site mirror of quay.io
 kolla_docker_registry: "{{ zuul_site_mirror_fqdn }}:4447"
+# TODO(priteau): Remove this when previous_release is Xena
+{% if is_previous_release | default(false) and previous_release == 'wallaby'%}
+# Keep the existing image type when installing Wallaby.
+kolla_install_type: "{{ 'source' if ansible_distribution == 'Ubuntu' else 'binary' }}"
+{% endif %}
 {% if not is_previous_release | default(false) %}
 kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}"
 kolla_source_version: "{{ zuul.projects['opendev.org/openstack/kolla'].checkout }}"
@@ -22,6 +27,8 @@ kolla_openstack_logging_debug: true
 # Use the CI infra's PyPI mirror.
 pip_local_mirror: true
 pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"
+pip_trusted_hosts:
+  - "{{ zuul_site_mirror_fqdn }}"
 
 # NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during ifdown,
 # and removes the bridge if there are no interfaces left. When Kayobe bounces

playbooks/kayobe-overcloud-upgrade-base/run.yml

@@ -10,6 +10,7 @@
           shell:
             cmd: dev/install.sh &> {{ logs_dir }}/ansible/install-pre-upgrade
             chdir: "{{ previous_kayobe_src_dir }}"
+            executable: /bin/bash
 
         - name: Configure the firewall
           # We run kayobe commands to determine variable values. So this must
@@ -19,15 +20,22 @@
             # TODO(mgoddard): Use the previous repo when the previous release
             # is no longer Wallaby.
             chdir: "{{ kayobe_src_dir }}"
+            executable: /bin/bash
 
         - name: Ensure overcloud is deployed
           shell:
             cmd: dev/overcloud-deploy.sh &> {{ logs_dir }}/ansible/overcloud-deploy-pre-upgrade
             chdir: "{{ previous_kayobe_src_dir }}"
+            executable: /bin/bash
 
       environment:
         KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}"
 
+    - name: Ensure qemu-img is available for Tenks
+      command: docker exec -u root nova_libvirt bash -c 'apt update && apt -y install qemu-utils'
+      become: true
+      when: ansible_os_family == 'Debian'
+
     # Update the Kayobe configuration to the current release.
 
     - name: Ensure kolla config directory exists
@@ -66,22 +74,32 @@
             # cause this to fail.
             cmd: dev/tenks-deploy-compute.sh '{{ tenks_src_dir }}' &> {{ logs_dir }}/ansible/tenks-deploy
             chdir: "{{ kayobe_src_dir }}"
+            executable: /bin/bash
 
         - name: Perform testing of the VMs in the overcloud prior to upgrade
           shell:
             cmd: dev/overcloud-test-vm.sh &> {{ logs_dir }}/ansible/overcloud-test-vm-pre-upgrade
             chdir: "{{ previous_kayobe_src_dir }}"
+            executable: /bin/bash
 
         - name: Perform testing of the baremetal machines in the overcloud prior to upgrade
           shell:
             cmd: dev/overcloud-test-baremetal.sh &> {{ logs_dir }}/ansible/overcloud-test-bm-pre-upgrade
             chdir: "{{ previous_kayobe_src_dir }}"
+            executable: /bin/bash
+          # FIXME(mgoddard): Bare metal testing is unreliable on Ubuntu - some jobs
+          # see IPMI failures such as the following:
+          # ipmitool chassis bootdev pxe
+          # Error setting Chassis Boot Parameter 5\nError setting Chassis Boot
+          # Parameter 0\n
+          when: ansible_os_family != 'Debian'
 
         # Upgrade Kayobe, and use it to perform an upgrade of the control plane.
 
         - name: Ensure overcloud is upgraded
           shell:
             cmd: "{{ kayobe_src_dir }}/dev/overcloud-upgrade.sh &> {{ logs_dir }}/ansible/overcloud-upgrade"
+            executable: /bin/bash
 
         # Remove unused Docker images to avoid reaching full disk
         - name: Remove unused Docker images
@@ -98,6 +116,7 @@
               source {{ kayobe_config_src_dir }}/etc/kolla/public-openrc.sh &&
               openstack baremetal node set tk0 --deploy-interface direct &&
               openstack baremetal node set tk1 --deploy-interface direct
+            executable: /bin/bash
 
         # Perform a smoke test against the upgraded current release.
 
@@ -105,11 +124,19 @@
           shell:
             cmd: dev/overcloud-test-vm.sh &> {{ logs_dir }}/ansible/overcloud-test-vm-post-upgrade
             chdir: "{{ kayobe_src_dir }}"
+            executable: /bin/bash
 
         - name: Perform testing of the baremetal machines in the upgraded overcloud
           shell:
             cmd: dev/overcloud-test-baremetal.sh &> {{ logs_dir }}/ansible/overcloud-test-bm-post-upgrade
             chdir: "{{ kayobe_src_dir }}"
+            executable: /bin/bash
+          # FIXME(mgoddard): Bare metal testing is unreliable on Ubuntu - some jobs
+          # see IPMI failures such as the following:
+          # ipmitool chassis bootdev pxe
+          # Error setting Chassis Boot Parameter 5\nError setting Chassis Boot
+          # Parameter 0\n
+          when: ansible_os_family != 'Debian'
 
       environment:
         KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"

playbooks/kayobe-seed-upgrade-base/overrides.yml.j2

@@ -22,6 +22,8 @@ kolla_openstack_logging_debug: True
 # Use the CI infra's PyPI mirror.
 pip_local_mirror: true
 pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"
+pip_trusted_hosts:
+  - "{{ zuul_site_mirror_fqdn }}"
 
 # NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during ifdown,
 # and removes the bridge if there are no interfaces left. When Kayobe bounces

playbooks/kayobe-seed-upgrade-base/run.yml

@@ -10,11 +10,13 @@
           shell:
             cmd: dev/install.sh &> {{ logs_dir }}/ansible/install-pre-upgrade
             chdir: "{{ previous_kayobe_src_dir }}"
+            executable: /bin/bash
 
         - name: Ensure seed is deployed
           shell:
             cmd: dev/seed-deploy.sh &> {{ logs_dir }}/ansible/seed-deploy-pre-upgrade
             chdir: "{{ previous_kayobe_src_dir }}"
+            executable: /bin/bash
       environment:
         KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}"
         # Don't provision a seed VM - use the Zuul VM as the seed host.
@@ -56,6 +58,7 @@
         - name: Ensure seed is upgraded
           shell:
             cmd: "{{ kayobe_src_dir }}/dev/seed-upgrade.sh &> {{ logs_dir }}/ansible/seed-upgrade"
+            executable: /bin/bash
 
         # TODO(mgoddard): Perform a smoke test against the upgraded current release.
       environment:

zuul.d/jobs.yaml

@@ -133,6 +133,11 @@
     parent: kayobe-overcloud-upgrade-base
     nodeset: kayobe-centos8s
 
+- job:
+    name: kayobe-overcloud-upgrade-ubuntu-focal
+    parent: kayobe-overcloud-upgrade-base
+    nodeset: kayobe-ubuntu-focal
+
 - job:
     name: kayobe-seed-base
     parent: kayobe-base
@@ -192,6 +197,11 @@
     parent: kayobe-seed-upgrade-base
     nodeset: kayobe-centos8s
 
+- job:
+    name: kayobe-seed-upgrade-ubuntu-focal
+    parent: kayobe-seed-upgrade-base
+    nodeset: kayobe-ubuntu-focal
+
 - job:
     name: kayobe-seed-vm-base
     parent: kayobe-base

zuul.d/project.yaml

@@ -16,9 +16,11 @@
         - kayobe-overcloud-host-configure-centos8s
         - kayobe-overcloud-host-configure-ubuntu-focal
         - kayobe-overcloud-upgrade-centos8s
+        - kayobe-overcloud-upgrade-ubuntu-focal
         - kayobe-seed-centos8s
         - kayobe-seed-ubuntu-focal
         - kayobe-seed-upgrade-centos8s
+        - kayobe-seed-upgrade-ubuntu-focal
         - kayobe-seed-vm-centos8s
         - kayobe-seed-vm-ubuntu-focal