Add seed VM provisioning CI job

Adds the kayobe-seed-vm-centos8 CI job to configure the Zuul VM as a
seed hypervisor, and use nested virt to provision a seed VM.  This
ensures that the seed hypervisor code paths are tested.

The job uses a Cirros image for the seed VM rather than the usual CentOS
cloud image. This is to reduce bandwidth required to download the image.
It does mean that the resulting seed VM cannot be used as a seed, but
nested virt would make this slow and unreliable anyway. Cirros does not
load cdrom drivers by default, so we add the configdrive as a disk
rather than a cdrom device.

Depends-On: https://review.opendev.org/617161

Change-Id: I2268a1ddf9a2870c713f32a40689e1686365aabd
Story: 2001655
Task: 6683

ansible/seed-vm-provision.yml

@@ -82,12 +82,13 @@
 
   roles:
     - role: stackhpc.libvirt-vm
+      seed_vm_configdrive_device: cdrom
       seed_vm_configdrive_volume:
         name: "{{ hostvars[seed_host].seed_vm_name }}-configdrive"
         pool: "{{ hostvars[seed_host].seed_vm_pool }}"
         # Round size up to next multiple of 4096.
         capacity: "{{ (stat_result.stat.size + 4095) // 4096 * 4096 }}"
-        device: "cdrom"
+        device: "{{ seed_vm_configdrive_device }}"
         format: "raw"
         image: "{{ seed_vm_configdrive_path }}"
       libvirt_vm_image_cache_path: "{{ image_cache_path }}"

dev/functions

@@ -30,10 +30,16 @@ function config_defaults {
     # Whether to provision a VM for the seed host.
     export KAYOBE_SEED_VM_PROVISION=${KAYOBE_SEED_VM_PROVISION:-1}
 
+    # Whether to configure the seed host.
+    export KAYOBE_SEED_HOST_CONFIGURE=${KAYOBE_SEED_HOST_CONFIGURE:-1}
+
     # Whether to build container images for the seed services. If 0, they will
     # be pulled.
     export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=${KAYOBE_SEED_CONTAINER_IMAGE_BUILD:-0}
 
+    # Whether to deploy seed services.
+    export KAYOBE_SEED_SERVICE_DEPLOY=${KAYOBE_SEED_SERVICE_DEPLOY:-1}
+
     # Whether to build container images for the overcloud services. If 0, they
     # will be pulled if $KAYOBE_OVERCLOUD_CONTAINER_IMAGE_PULL is 1.
     export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD:-0}
@@ -262,8 +268,10 @@ function seed_deploy {
         run_kayobe seed vm provision
     fi
 
+    if [[ ${KAYOBE_SEED_HOST_CONFIGURE} = 1 ]]; then
         echo "Configuring the seed host"
         run_kayobe seed host configure
+    fi
 
     # Note: This must currently be done before host configure, because host
     # configure runs kolla-ansible.yml, which validates the presence of the
@@ -283,8 +291,10 @@ function seed_deploy {
         #run_kayobe seed container image pull
     fi
 
+    if [[ ${KAYOBE_SEED_SERVICE_DEPLOY} = 1 ]]; then
         echo "Deploying containerised seed services"
         run_kayobe seed service deploy
+    fi
 }
 
 function seed_upgrade {

playbooks/kayobe-seed-vm-base/overrides.yml.j2

@@ -0,0 +1,44 @@
+---
+# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
+# Ansible is run directly on the controller.
+disable_selinux_do_reboot: false
+
+# Use the OpenStack infra's Dockerhub mirror.
+docker_registry_mirrors:
+  - "http://{{ zuul_site_mirror_fqdn }}:8082/"
+
+kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}"
+kolla_source_version: "{{ zuul.projects['opendev.org/openstack/kolla'].checkout }}"
+kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla-ansible'].src_dir }}"
+kolla_ansible_source_version: "{{ zuul.projects['opendev.org/openstack/kolla-ansible'].checkout }}"
+kolla_openstack_logging_debug: True
+pip_upper_constraints_file: "/tmp/upper-constraints.txt"
+
+# The hosts used by Zuul may or may not have Virtualization Technology (VT)
+# enabled. Don't fail if it's disabled.
+libvirt_host_require_vt: false
+
+# Use the CI infra's PyPI mirror.
+pip_local_mirror: true
+pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"
+pip_trusted_hosts:
+  - "{{ zuul_site_mirror_fqdn }}"
+
+# Try with only a single VCPU, word on the street is that QEMU doesn't play
+# nicely with more than one.
+seed_vm_vcpus: 1
+
+# Reduce the memory footprint of the seed VM.
+seed_vm_memory_mb: "{{ 1 * 1024 }}"
+
+# Use cirros rather than CentOS for the VM.
+seed_bootstrap_user: cirros
+seed_vm_root_image: https://download.cirros-cloud.net/0.5.1/cirros-0.5.1-x86_64-disk.img
+
+# Cirros doesn't load cdom drivers by default.
+seed_vm_configdrive_device: disk
+
+# Cirros is Debian family, but doesn't support path globs in
+# /etc/network/interfaces.
+configdrive_os_family: Debian
+configdrive_debian_network_interfaces_supports_glob: false

playbooks/kayobe-seed-vm-base/pre.yml

@@ -0,0 +1,63 @@
+---
+- hosts: primary
+  environment:
+    KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
+  tasks:
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+
+    - name: Ensure seed group variables exist
+      template:
+        src: seed-group-vars.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/inventory/group_vars/seed/network-interfaces"
+
+    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+    # braio - to exist with an IP address of 192.168.33.4.
+    - name: Ensure all-in-one network bridge interface exists
+      command: "{{ item }}"
+      become: true
+      with_items:
+        - "ip l add braio type bridge"
+        - "ip l set braio up"
+        - "ip a add 192.168.33.4/24 dev braio"
+        # NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during
+        # ifdown, and removes the bridge if there are no interfaces left. When
+        # Kayobe bounces veth links plugged into the bridge, it causes the
+        # bridge which has the IP we are using for SSH to be removed. Use a
+        # dummy interface.
+        - "ip l add dummy1 type dummy"
+        - "ip l set dummy1 up"
+        - "ip l set dummy1 master braio"
+
+    # NOTE(mgoddard): Configure IP forwarding and NAT to allow communication
+    # from the seed VM to the outside world.
+
+    # FIXME(mgoddard): use a libvirt network?
+    - name: Ensure NAT is configured
+      iptables:
+        chain: POSTROUTING
+        table: nat
+        out_interface: "{{ ansible_default_ipv4.interface }}"
+        jump: MASQUERADE
+      become: true
+
+    # FIXME(mgoddard): use a libvirt network?
+    - name: Ensure IP forwarding is enabled
+      sysctl:
+        name: net.ipv4.conf.all.forwarding
+        value: 1
+      become: true
+
+    - name: Ensure SELinux is disabled
+      selinux:
+        state: disabled
+      become: True
+
+    - name: Ensure kayobe is installed
+      shell:
+        cmd: dev/install.sh &> {{ logs_dir }}/ansible/install
+        chdir: "{{ kayobe_src_dir }}"

playbooks/kayobe-seed-vm-base/run.yml

@@ -0,0 +1,15 @@
+---
+- hosts: primary
+  environment:
+    KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
+    # The CirrOS image does not support much beyond logging in.
+    KAYOBE_SEED_HOST_CONFIGURE: 0
+    KAYOBE_SEED_SERVICE_DEPLOY: 0
+  tasks:
+    - name: Ensure seed hypervisor is deployed
+      shell:
+        cmd: "{{ kayobe_src_dir }}/dev/seed-hypervisor-deploy.sh > {{ logs_dir }}/ansible/seed-hypervisor-deploy"
+
+    - name: Ensure seed is deployed
+      shell:
+        cmd: "{{ kayobe_src_dir }}/dev/seed-deploy.sh &> {{ logs_dir }}/ansible/seed-deploy"

playbooks/kayobe-seed-vm-base/seed-group-vars.j2

@@ -0,0 +1,5 @@
+---
+aio_interface: eth0
+
+# Route via the seed-hypervisor to the outside world.
+aio_gateway: 192.168.33.4

requirements.yml

@@ -3,7 +3,7 @@
   version: 1.3.1
 - src: jriguera.configdrive
   # There are no versioned releases of this role.
-  version: 56daf0017783bd4fa5722da4ee91b66ddd4e6b6a
+  version: 8438592c84585c86e62ae07e526d3da53629b377
 - src: MichaelRigart.interfaces
   version: v1.5.0
 - src: mrlesmithjr.manage-lvm

zuul.d/jobs.yaml

@@ -165,3 +165,20 @@
     name: kayobe-seed-upgrade-centos8
     parent: kayobe-seed-upgrade-base
     nodeset: kayobe-centos8
+
+- job:
+    name: kayobe-seed-vm-base
+    parent: kayobe-base
+    description: |
+      Base job for testing seed VM provisioning.
+
+      Configures the primary VM as a libvirt hypervisor, and provisions a seed
+      as a VM.
+    pre-run: playbooks/kayobe-seed-vm-base/pre.yml
+    run: playbooks/kayobe-seed-vm-base/run.yml
+    timeout: 5400
+
+- job:
+    name: kayobe-seed-vm-centos8
+    parent: kayobe-seed-vm-base
+    nodeset: kayobe-centos8

zuul.d/project.yaml

@@ -15,6 +15,7 @@
         - kayobe-overcloud-upgrade-centos8
         - kayobe-seed-centos8
         - kayobe-seed-upgrade-centos8
+        - kayobe-seed-vm-centos8
 
     gate:
       queue: kayobe
@@ -27,3 +28,4 @@
         - kayobe-overcloud-upgrade-centos8
         - kayobe-seed-centos8
         - kayobe-seed-upgrade-centos8
+        - kayobe-seed-vm-centos8