Add a job for seed deployment

Adds a new CI job, kayobe-seed-centos, that deploys a single VM as a
seed host. The VM provided by Zuul is configured as the seed, rather
than a seed hypervisor that provisions a seed VM using nested virt.

This approach should be more reliable, faster, and easier to get
working, at the expense of not testing the seed hypervisor code paths.

Currently the seed services are deployed, but not tested. A future
change could test discovery and provisioning of an overcloud host using
the seed.

Change-Id: Iefe16215af6f1a9fb8ec78094cd6bd76a82a040a
Story: 2001655
Task: 6683

dev/config.sh

@@ -10,6 +10,9 @@
 # Path to the kayobe virtual environment.
 #export KAYOBE_VENV_PATH=~/kayobe-venv
 
+# Whether to provision a VM for the seed host.
+#export KAYOBE_SEED_VM_PROVISION=1
+
 # Whether to build container images for the seed services. If 0, they will be
 # pulled.
 #export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=0
@@ -17,3 +20,6 @@
 # Whether to build container images for the overcloud services. If 0, they will
 # be pulled.
 #export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=0
+
+# Additional arguments to pass to kayobe commands.
+#export KAYOBE_EXTRA_ARGS=

dev/functions

@@ -27,6 +27,9 @@ function config_defaults {
     # Path to the kayobe virtual environment.
     export KAYOBE_VENV_PATH="${KAYOBE_VENV_PATH:-${HOME}/kayobe-venv}"
 
+    # Whether to provision a VM for the seed host.
+    export KAYOBE_SEED_VM_PROVISION=${KAYOBE_SEED_VM_PROVISION:-1}
+
     # Whether to build container images for the seed services. If 0, they will
     # be pulled.
     export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=${KAYOBE_SEED_CONTAINER_IMAGE_BUILD:-0}
@@ -34,6 +37,9 @@ function config_defaults {
     # Whether to build container images for the overcloud services. If 0, they
     # will be pulled.
     export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD:-0}
+
+    # Additional arguments to pass to kayobe commands.
+    export KAYOBE_EXTRA_ARGS=${KAYOBE_EXTRA_ARGS:-}
 }
 
 function config_set {
@@ -115,15 +121,21 @@ function environment_setup {
     cd "${KAYOBE_SOURCE_PATH}"
 }
 
+function run_kayobe {
+    # Run a kayobe command, including extra arguments provided via
+    # $KAYOBE_EXTRA_ARGS.
+    kayobe ${KAYOBE_EXTRA_ARGS} $*
+}
+
 function seed_hypervisor_deploy {
     # Deploy a seed hypervisor.
     environment_setup
 
     echo "Bootstrapping the ansible control host"
-    kayobe control host bootstrap
+    run_kayobe control host bootstrap
 
     echo "Configuring the seed hypervisor"
-    kayobe seed hypervisor host configure
+    run_kayobe seed hypervisor host configure
 }
 
 function seed_deploy {
@@ -131,34 +143,36 @@ function seed_deploy {
     environment_setup
 
     echo "Bootstrapping the ansible control host"
-    kayobe control host bootstrap
+    run_kayobe control host bootstrap
 
+    if [[ ${KAYOBE_SEED_VM_PROVISION} = 1 ]]; then
         echo "Provisioning the seed VM"
-    kayobe seed vm provision
+        run_kayobe seed vm provision
+    fi
 
     echo "Configuring the seed host"
-    kayobe seed host configure
+    run_kayobe seed host configure
 
     # Note: This must currently be before host configure, because host
     # configure runs kolla-ansible.yml, which validates the presence of the
     # built deploy images.
     if is_deploy_image_built_locally; then
         echo "Building seed deployment images"
-        kayobe seed deployment image build
+        run_kayobe seed deployment image build
     else
         echo "Not building seed deployment images"
     fi
 
     if [[ ${KAYOBE_SEED_CONTAINER_IMAGE_BUILD} = 1 ]]; then
         echo "Building seed container images"
-        kayobe seed container image build
+        run_kayobe seed container image build
     else
         echo "Not pulling seed container images - no such command yet"
-        #kayobe seed container image pull
+        #run_kayobe seed container image pull
     fi
 
     echo "Deploying containerised seed services"
-    kayobe seed service deploy
+    run_kayobe seed service deploy
 }
 
 function overcloud_deploy {
@@ -169,35 +183,35 @@ function overcloud_deploy {
     environment_setup
 
     echo "Bootstrapping the ansible control host"
-    kayobe control host bootstrap
+    run_kayobe control host bootstrap
 
     echo "Configuring the controller host"
-    kayobe overcloud host configure
+    run_kayobe overcloud host configure
 
     # Note: This must currently be before host configure, because host
     # configure runs kolla-ansible.yml, which validates the presence of the
     # built deploy images.
     if is_deploy_image_built_locally; then
         echo "Building overcloud deployment images"
-        kayobe overcloud deployment image build
+        run_kayobe overcloud deployment image build
     else
         echo "Not building overcloud deployment images"
     fi
 
     if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then
         echo "Building overcloud container images"
-        kayobe overcloud container image build
+        run_kayobe overcloud container image build
     else
         echo "Pulling overcloud container images"
-        kayobe overcloud container image pull
+        run_kayobe overcloud container image pull
     fi
 
     echo "Deploying containerised overcloud services"
-    kayobe overcloud service deploy
+    run_kayobe overcloud service deploy
 
     echo "Performing post-deployment configuration"
     source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh"
-    kayobe overcloud post configure
+    run_kayobe overcloud post configure
 
     echo "Control plane deployment complete"
 }

kayobe/ansible.py

@@ -99,7 +99,10 @@ def _validate_args(parsed_args, playbooks):
 
 
 def _get_vars_files(config_path):
-    """Return a list of Kayobe Ansible configuration variable files."""
+    """Return a list of Kayobe Ansible configuration variable files.
+
+    The files will be sorted alphabetically by name.
+    """
     vars_files = []
     for vars_file in os.listdir(config_path):
         abs_path = os.path.join(config_path, vars_file)
@@ -107,7 +110,7 @@ def _get_vars_files(config_path):
             root, ext = os.path.splitext(vars_file)
             if ext in (".yml", ".yaml", ".json"):
                 vars_files.append(abs_path)
-    return vars_files
+    return sorted(vars_files)
 
 
 def build_args(parsed_args, playbooks,

playbooks/kayobe-overcloud-base/pre.yml

@@ -3,6 +3,7 @@
   vars:
     logs_dir: "/tmp/logs"
     kayobe_src_dir: "{{ zuul.project.src_dir }}"
+    kayobe_config_src_dir: "{{ kayobe_src_dir }}/config/src/kayobe-config"
   roles:
     - role: kayobe-diagnostics
       kayobe_diagnostics_phase: "pre"
@@ -31,18 +32,20 @@
 
     - name: Ensure kayobe-config directory exists
       file:
-        path: "{{ kayobe_src_dir }}/config/src"
+        path: "{{ kayobe_config_src_dir }}"
         state: directory
 
     - name: Ensure kayobe-config repository is cloned
       git:
         repo: https://github.com/stackhpc/dev-kayobe-config
-        dest: "{{ kayobe_src_dir }}/config/src/kayobe-config"
+        dest: "{{ kayobe_config_src_dir }}"
 
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
     - name: Ensure kayobe-config override config file exists
       template:
         src: overrides.yml.j2
-        dest: "{{ kayobe_src_dir }}/config/src/kayobe-config/etc/kayobe/overrides.yml"
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
 
     # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
     # breth1 - to exist with an IP address of 192.168.33.3.

playbooks/kayobe-seed-base/bifrost-overrides.yml.j2

@@ -0,0 +1,7 @@
+---
+# Don't build an IPA deployment image, instead download upstream images.
+create_ipa_image: false
+download_ipa: true
+
+# Don't build a disk image. It takes time and can be unreliable.
+use_cirros: true

playbooks/kayobe-seed-base/overrides.yml.j2

@@ -0,0 +1,14 @@
+---
+# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
+# Ansible is run directly on the controller.
+disable_selinux_do_reboot: false
+
+# NOTE(mgoddard): We're using a cirros image, which doesn't require the
+# resolv.conf work around used for CentOS.
+overcloud_host_image_workaround_resolv_enabled: false
+
+# NOTE(mgoddard): Use a loopback-mounted LVM volume for docker storage since
+# the overlay driver doesn't work with the ansible template module until
+# ansible 2.4.0, and this is required by bifrost.
+seed_lvm_group_data_disks:
+  - /dev/loop0

playbooks/kayobe-seed-base/post.yml

@@ -0,0 +1,7 @@
+---
+- hosts: all
+  roles:
+    - role: kayobe-diagnostics
+      kayobe_diagnostics_phase: "post"
+      kayobe_diagnostics_log_dir: "/tmp/logs"
+      kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"

playbooks/kayobe-seed-base/pre.yml

@@ -0,0 +1,84 @@
+---
+- hosts: primary
+  vars:
+    logs_dir: "/tmp/logs"
+    kayobe_src_dir: "{{ zuul.project.src_dir }}"
+    kayobe_config_src_dir: "{{ kayobe_src_dir }}/config/src/kayobe-config"
+  roles:
+    - role: kayobe-diagnostics
+      kayobe_diagnostics_phase: "pre"
+      kayobe_diagnostics_log_dir: "{{ logs_dir }}"
+  tasks:
+    - name: Install dbus for debian system
+      apt:
+        name: dbus
+      when:
+        - ansible_os_family == 'Debian'
+      become: true
+
+    - block:
+        # NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
+        # but the configure-mirrors role used by Zuul disables epel. Since we
+        # install epel-release and expect epel to be enabled, enable it here.
+        - name: Ensure yum-utils is installed
+          yum:
+            name: yum-utils
+            state: installed
+
+        - name: Enable the EPEL yum repository
+          command: yum-config-manager --enable epel
+      when: ansible_os_family == 'RedHat'
+      become: true
+
+    - name: Ensure kayobe-config directory exists
+      file:
+        path: "{{ kayobe_config_src_dir }}"
+        state: directory
+
+    - name: Ensure kayobe-config repository is cloned
+      git:
+        repo: https://github.com/stackhpc/dev-kayobe-config
+        dest: "{{ kayobe_config_src_dir }}"
+
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+
+    - name: Ensure bifrost overrides directory exists
+      file:
+        path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost"
+        state: "directory"
+
+    - name: Ensure bifrost overrides file exists
+      template:
+        src: bifrost-overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/bifrost.yml"
+
+    # NOTE(mgoddard): Create a loopback device backed by a file for docker
+    # storage.  We do this since the overlay driver doesn't work with the
+    # ansible template module until ansible 2.4.0, and this is required by
+    # bifrost.
+    - name: Ensure a docker storage backing file exists
+      command: truncate -s 20G /tmp/docker-storage
+
+    - name: Ensure the docker storage loopback device is created
+      command: losetup /dev/loop0 /tmp/docker-storage
+      become: true
+
+    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+    # breth1 - to exist on the seed with an IP address of 192.168.33.5.
+    - name: Ensure all-in-one network bridge interface exists
+      command: "{{ item }}"
+      become: true
+      with_items:
+        - "ip l add breth1 type bridge"
+        - "ip l set breth1 up"
+        - "ip a add 192.168.33.5/24 dev breth1"
+
+    - name: Ensure kayobe is installed
+      shell:
+        cmd: dev/install.sh > {{ logs_dir }}/ansible/install
+        chdir: "{{ kayobe_src_dir }}"

playbooks/kayobe-seed-base/run.yml

@@ -0,0 +1,13 @@
+---
+- hosts: primary
+  vars:
+    kayobe_src_dir: "{{ zuul.project.src_dir }}"
+    logs_dir: "/tmp/logs"
+  tasks:
+    - name: Ensure seed is deployed
+      shell:
+        cmd: dev/seed-deploy.sh > {{ logs_dir }}/ansible/seed-deploy
+        chdir: "{{ kayobe_src_dir }}"
+      environment:
+        # Don't provision a seed VM - use the Zuul VM as the seed host.
+        KAYOBE_SEED_VM_PROVISION: 0

roles/kayobe-diagnostics/files/get_logs.sh

@@ -28,6 +28,8 @@ copy_logs() {
     parted -l > ${LOG_DIR}/system_logs/parted-l.txt
     mount > ${LOG_DIR}/system_logs/mount.txt
     env > ${LOG_DIR}/system_logs/env.txt
+    ip address > ${LOG_DIR}/system_logs/ip-address.txt
+    ip route > ${LOG_DIR}/system_logs/ip-route.txt
 
     if [ `command -v dpkg` ]; then
         dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt
@@ -50,13 +52,6 @@ copy_logs() {
     # logs.openstack.org clicking results in the browser shows the
     # files, rather than trying to send it to another app or make you
     # download it, etc.
-
-    # Rename files to .txt; this is so that when displayed via
-    # logs.openstack.org clicking results in the browser shows the
-    # files, rather than trying to send it to another app or make you
-    # download it, etc.
-
-    # Rename all .log files to .txt files
     for f in $(find ${LOG_DIR}/{system_logs,kolla,docker_logs} -name "*.log"); do
         mv $f ${f/.log/.txt}
     done

roles/kayobe-diagnostics/tasks/post.yml

@@ -16,6 +16,6 @@
 
 - name: Download logs to executor
   synchronize:
-    src: "{{ kayobe_diagnostics_log_dir }}"
+    src: "{{ kayobe_diagnostics_log_dir }}/"
     dest: "{{ kayobe_diagnostics_executor_log_dir }}/"
     mode: pull

zuul.d/jobs.yaml

@@ -16,6 +16,10 @@
 # Base job for testing overcloud deployment.
 - job:
     name: kayobe-overcloud-base
+    description: |
+      Base job for testing overcloud deployment.
+
+      Configures the primary VM as an overcloud controller.
     pre-run: playbooks/kayobe-overcloud-base/pre.yml
     run: playbooks/kayobe-overcloud-base/run.yml
     post-run: playbooks/kayobe-overcloud-base/post.yml
@@ -29,3 +33,23 @@
     name: kayobe-overcloud-centos
     parent: kayobe-overcloud-base
     nodeset: kayobe-centos
+
+- job:
+    name: kayobe-seed-base
+    description: |
+      Base job for testing seed deployment.
+
+      Configures the primary VM as a seed.
+    pre-run: playbooks/kayobe-seed-base/pre.yml
+    run: playbooks/kayobe-seed-base/run.yml
+    post-run: playbooks/kayobe-seed-base/post.yml
+    attempts: 1
+    timeout: 5400
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*
+
+- job:
+    name: kayobe-seed-centos
+    parent: kayobe-seed-base
+    nodeset: kayobe-centos

zuul.d/project.yaml

@@ -10,6 +10,7 @@
         - kayobe-tox-ansible-syntax
         - kayobe-tox-ansible
         - kayobe-overcloud-centos
+        - kayobe-seed-centos
 
     gate:
       queue: kayobe
@@ -22,3 +23,4 @@
         - kayobe-tox-ansible-syntax
         - kayobe-tox-ansible
         - kayobe-overcloud-centos
+        - kayobe-seed-centos