Merge "Add a job for seed deployment"

dev/config.sh

@@ -10,6 +10,9 @@
 # Path to the kayobe virtual environment.
 #export KAYOBE_VENV_PATH=~/kayobe-venv
 
+# Whether to provision a VM for the seed host.
+#export KAYOBE_SEED_VM_PROVISION=1
+
 # Whether to build container images for the seed services. If 0, they will be
 # pulled.
 #export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=0
@@ -17,3 +20,6 @@
 # Whether to build container images for the overcloud services. If 0, they will
 # be pulled.
 #export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=0
+
+# Additional arguments to pass to kayobe commands.
+#export KAYOBE_EXTRA_ARGS=

dev/functions

@@ -27,6 +27,9 @@ function config_defaults {
     # Path to the kayobe virtual environment.
     export KAYOBE_VENV_PATH="${KAYOBE_VENV_PATH:-${HOME}/kayobe-venv}"
 
+    # Whether to provision a VM for the seed host.
+    export KAYOBE_SEED_VM_PROVISION=${KAYOBE_SEED_VM_PROVISION:-1}
+
     # Whether to build container images for the seed services. If 0, they will
     # be pulled.
     export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=${KAYOBE_SEED_CONTAINER_IMAGE_BUILD:-0}
@@ -34,6 +37,9 @@ function config_defaults {
     # Whether to build container images for the overcloud services. If 0, they
     # will be pulled.
     export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD:-0}
+
+    # Additional arguments to pass to kayobe commands.
+    export KAYOBE_EXTRA_ARGS=${KAYOBE_EXTRA_ARGS:-}
 }
 
 function config_set {
@@ -115,15 +121,21 @@ function environment_setup {
     cd "${KAYOBE_SOURCE_PATH}"
 }
 
+function run_kayobe {
+    # Run a kayobe command, including extra arguments provided via
+    # $KAYOBE_EXTRA_ARGS.
+    kayobe ${KAYOBE_EXTRA_ARGS} $*
+}
+
 function seed_hypervisor_deploy {
     # Deploy a seed hypervisor.
     environment_setup
 
     echo "Bootstrapping the ansible control host"
-    kayobe control host bootstrap
+    run_kayobe control host bootstrap
 
     echo "Configuring the seed hypervisor"
-    kayobe seed hypervisor host configure
+    run_kayobe seed hypervisor host configure
 }
 
 function seed_deploy {
@@ -131,34 +143,36 @@ function seed_deploy {
     environment_setup
 
     echo "Bootstrapping the ansible control host"
-    kayobe control host bootstrap
+    run_kayobe control host bootstrap
 
-    echo "Provisioning the seed VM"
-    kayobe seed vm provision
+    if [[ ${KAYOBE_SEED_VM_PROVISION} = 1 ]]; then
+        echo "Provisioning the seed VM"
+        run_kayobe seed vm provision
+    fi
 
     echo "Configuring the seed host"
-    kayobe seed host configure
+    run_kayobe seed host configure
 
     # Note: This must currently be before host configure, because host
     # configure runs kolla-ansible.yml, which validates the presence of the
     # built deploy images.
     if is_deploy_image_built_locally; then
         echo "Building seed deployment images"
-        kayobe seed deployment image build
+        run_kayobe seed deployment image build
     else
         echo "Not building seed deployment images"
     fi
 
     if [[ ${KAYOBE_SEED_CONTAINER_IMAGE_BUILD} = 1 ]]; then
         echo "Building seed container images"
-        kayobe seed container image build
+        run_kayobe seed container image build
     else
         echo "Not pulling seed container images - no such command yet"
-        #kayobe seed container image pull
+        #run_kayobe seed container image pull
     fi
 
     echo "Deploying containerised seed services"
-    kayobe seed service deploy
+    run_kayobe seed service deploy
 }
 
 function overcloud_deploy {
@@ -169,35 +183,35 @@ function overcloud_deploy {
     environment_setup
 
     echo "Bootstrapping the ansible control host"
-    kayobe control host bootstrap
+    run_kayobe control host bootstrap
 
     echo "Configuring the controller host"
-    kayobe overcloud host configure
+    run_kayobe overcloud host configure
 
     # Note: This must currently be before host configure, because host
     # configure runs kolla-ansible.yml, which validates the presence of the
     # built deploy images.
     if is_deploy_image_built_locally; then
         echo "Building overcloud deployment images"
-        kayobe overcloud deployment image build
+        run_kayobe overcloud deployment image build
     else
         echo "Not building overcloud deployment images"
     fi
 
     if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then
         echo "Building overcloud container images"
-        kayobe overcloud container image build
+        run_kayobe overcloud container image build
     else
         echo "Pulling overcloud container images"
-        kayobe overcloud container image pull
+        run_kayobe overcloud container image pull
     fi
 
     echo "Deploying containerised overcloud services"
-    kayobe overcloud service deploy
+    run_kayobe overcloud service deploy
 
     echo "Performing post-deployment configuration"
     source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh"
-    kayobe overcloud post configure
+    run_kayobe overcloud post configure
 
     echo "Control plane deployment complete"
 }

kayobe/ansible.py

@@ -99,7 +99,10 @@ def _validate_args(parsed_args, playbooks):
 
 
 def _get_vars_files(config_path):
-    """Return a list of Kayobe Ansible configuration variable files."""
+    """Return a list of Kayobe Ansible configuration variable files.
+
+    The files will be sorted alphabetically by name.
+    """
     vars_files = []
     for vars_file in os.listdir(config_path):
         abs_path = os.path.join(config_path, vars_file)
@@ -107,7 +110,7 @@ def _get_vars_files(config_path):
             root, ext = os.path.splitext(vars_file)
             if ext in (".yml", ".yaml", ".json"):
                 vars_files.append(abs_path)
-    return vars_files
+    return sorted(vars_files)
 
 
 def build_args(parsed_args, playbooks,

playbooks/kayobe-overcloud-base/pre.yml

@@ -3,6 +3,7 @@
   vars:
     logs_dir: "/tmp/logs"
     kayobe_src_dir: "{{ zuul.project.src_dir }}"
+    kayobe_config_src_dir: "{{ kayobe_src_dir }}/config/src/kayobe-config"
   roles:
     - role: kayobe-diagnostics
       kayobe_diagnostics_phase: "pre"
@@ -31,18 +32,20 @@
 
     - name: Ensure kayobe-config directory exists
       file:
-        path: "{{ kayobe_src_dir }}/config/src"
+        path: "{{ kayobe_config_src_dir }}"
         state: directory
 
     - name: Ensure kayobe-config repository is cloned
       git:
         repo: https://github.com/stackhpc/dev-kayobe-config
-        dest: "{{ kayobe_src_dir }}/config/src/kayobe-config"
+        dest: "{{ kayobe_config_src_dir }}"
 
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
     - name: Ensure kayobe-config override config file exists
       template:
         src: overrides.yml.j2
-        dest: "{{ kayobe_src_dir }}/config/src/kayobe-config/etc/kayobe/overrides.yml"
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
 
     # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
     # breth1 - to exist with an IP address of 192.168.33.3.

playbooks/kayobe-seed-base/bifrost-overrides.yml.j2

@@ -0,0 +1,7 @@
+---
+# Don't build an IPA deployment image, instead download upstream images.
+create_ipa_image: false
+download_ipa: true
+
+# Don't build a disk image. It takes time and can be unreliable.
+use_cirros: true

playbooks/kayobe-seed-base/overrides.yml.j2

@@ -0,0 +1,14 @@
+---
+# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
+# Ansible is run directly on the controller.
+disable_selinux_do_reboot: false
+
+# NOTE(mgoddard): We're using a cirros image, which doesn't require the
+# resolv.conf work around used for CentOS.
+overcloud_host_image_workaround_resolv_enabled: false
+
+# NOTE(mgoddard): Use a loopback-mounted LVM volume for docker storage since
+# the overlay driver doesn't work with the ansible template module until
+# ansible 2.4.0, and this is required by bifrost.
+seed_lvm_group_data_disks:
+  - /dev/loop0

playbooks/kayobe-seed-base/post.yml

@@ -0,0 +1,7 @@
+---
+- hosts: all
+  roles:
+    - role: kayobe-diagnostics
+      kayobe_diagnostics_phase: "post"
+      kayobe_diagnostics_log_dir: "/tmp/logs"
+      kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"

playbooks/kayobe-seed-base/pre.yml

@@ -0,0 +1,84 @@
+---
+- hosts: primary
+  vars:
+    logs_dir: "/tmp/logs"
+    kayobe_src_dir: "{{ zuul.project.src_dir }}"
+    kayobe_config_src_dir: "{{ kayobe_src_dir }}/config/src/kayobe-config"
+  roles:
+    - role: kayobe-diagnostics
+      kayobe_diagnostics_phase: "pre"
+      kayobe_diagnostics_log_dir: "{{ logs_dir }}"
+  tasks:
+    - name: Install dbus for debian system
+      apt:
+        name: dbus
+      when:
+        - ansible_os_family == 'Debian'
+      become: true
+
+    - block:
+        # NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
+        # but the configure-mirrors role used by Zuul disables epel. Since we
+        # install epel-release and expect epel to be enabled, enable it here.
+        - name: Ensure yum-utils is installed
+          yum:
+            name: yum-utils
+            state: installed
+
+        - name: Enable the EPEL yum repository
+          command: yum-config-manager --enable epel
+      when: ansible_os_family == 'RedHat'
+      become: true
+
+    - name: Ensure kayobe-config directory exists
+      file:
+        path: "{{ kayobe_config_src_dir }}"
+        state: directory
+
+    - name: Ensure kayobe-config repository is cloned
+      git:
+        repo: https://github.com/stackhpc/dev-kayobe-config
+        dest: "{{ kayobe_config_src_dir }}"
+
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+
+    - name: Ensure bifrost overrides directory exists
+      file:
+        path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost"
+        state: "directory"
+
+    - name: Ensure bifrost overrides file exists
+      template:
+        src: bifrost-overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/bifrost.yml"
+
+    # NOTE(mgoddard): Create a loopback device backed by a file for docker
+    # storage.  We do this since the overlay driver doesn't work with the
+    # ansible template module until ansible 2.4.0, and this is required by
+    # bifrost.
+    - name: Ensure a docker storage backing file exists
+      command: truncate -s 20G /tmp/docker-storage
+
+    - name: Ensure the docker storage loopback device is created
+      command: losetup /dev/loop0 /tmp/docker-storage
+      become: true
+
+    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+    # breth1 - to exist on the seed with an IP address of 192.168.33.5.
+    - name: Ensure all-in-one network bridge interface exists
+      command: "{{ item }}"
+      become: true
+      with_items:
+        - "ip l add breth1 type bridge"
+        - "ip l set breth1 up"
+        - "ip a add 192.168.33.5/24 dev breth1"
+
+    - name: Ensure kayobe is installed
+      shell:
+        cmd: dev/install.sh > {{ logs_dir }}/ansible/install
+        chdir: "{{ kayobe_src_dir }}"

playbooks/kayobe-seed-base/run.yml

@@ -0,0 +1,13 @@
+---
+- hosts: primary
+  vars:
+    kayobe_src_dir: "{{ zuul.project.src_dir }}"
+    logs_dir: "/tmp/logs"
+  tasks:
+    - name: Ensure seed is deployed
+      shell:
+        cmd: dev/seed-deploy.sh > {{ logs_dir }}/ansible/seed-deploy
+        chdir: "{{ kayobe_src_dir }}"
+      environment:
+        # Don't provision a seed VM - use the Zuul VM as the seed host.
+        KAYOBE_SEED_VM_PROVISION: 0

roles/kayobe-diagnostics/files/get_logs.sh

@@ -28,6 +28,8 @@ copy_logs() {
     parted -l > ${LOG_DIR}/system_logs/parted-l.txt
     mount > ${LOG_DIR}/system_logs/mount.txt
     env > ${LOG_DIR}/system_logs/env.txt
+    ip address > ${LOG_DIR}/system_logs/ip-address.txt
+    ip route > ${LOG_DIR}/system_logs/ip-route.txt
 
     if [ `command -v dpkg` ]; then
         dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt
@@ -50,13 +52,6 @@ copy_logs() {
     # logs.openstack.org clicking results in the browser shows the
     # files, rather than trying to send it to another app or make you
     # download it, etc.
-
-    # Rename files to .txt; this is so that when displayed via
-    # logs.openstack.org clicking results in the browser shows the
-    # files, rather than trying to send it to another app or make you
-    # download it, etc.
-
-    # Rename all .log files to .txt files
     for f in $(find ${LOG_DIR}/{system_logs,kolla,docker_logs} -name "*.log"); do
         mv $f ${f/.log/.txt}
     done

roles/kayobe-diagnostics/tasks/post.yml

@@ -16,6 +16,6 @@
 
 - name: Download logs to executor
   synchronize:
-    src: "{{ kayobe_diagnostics_log_dir }}"
+    src: "{{ kayobe_diagnostics_log_dir }}/"
     dest: "{{ kayobe_diagnostics_executor_log_dir }}/"
     mode: pull

zuul.d/jobs.yaml

@@ -16,6 +16,10 @@
 # Base job for testing overcloud deployment.
 - job:
     name: kayobe-overcloud-base
+    description: |
+      Base job for testing overcloud deployment.
+
+      Configures the primary VM as an overcloud controller.
     pre-run: playbooks/kayobe-overcloud-base/pre.yml
     run: playbooks/kayobe-overcloud-base/run.yml
     post-run: playbooks/kayobe-overcloud-base/post.yml
@@ -29,3 +33,23 @@
     name: kayobe-overcloud-centos
     parent: kayobe-overcloud-base
     nodeset: kayobe-centos
+
+- job:
+    name: kayobe-seed-base
+    description: |
+      Base job for testing seed deployment.
+
+      Configures the primary VM as a seed.
+    pre-run: playbooks/kayobe-seed-base/pre.yml
+    run: playbooks/kayobe-seed-base/run.yml
+    post-run: playbooks/kayobe-seed-base/post.yml
+    attempts: 1
+    timeout: 5400
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*
+
+- job:
+    name: kayobe-seed-centos
+    parent: kayobe-seed-base
+    nodeset: kayobe-centos

zuul.d/project.yaml

@@ -10,6 +10,7 @@
         - kayobe-tox-ansible-syntax
         - kayobe-tox-ansible
         - kayobe-overcloud-centos
+        - kayobe-seed-centos
 
     gate:
       queue: kayobe
@@ -22,3 +23,4 @@
         - kayobe-tox-ansible-syntax
         - kayobe-tox-ansible
         - kayobe-overcloud-centos
+        - kayobe-seed-centos