From cd6c366cc63cf6b34b9a9427bd339e6b949938ef Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 16 Mar 2018 12:26:18 +0000 Subject: [PATCH] Add a job for seed deployment Adds a new CI job, kayobe-seed-centos, that deploys a single VM as a seed host. The VM provided by Zuul is configured as the seed, rather than a seed hypervisor that provisions a seed VM using nested virt. This approach should be more reliable, faster, and easier to get working, at the expense of not testing the seed hypervisor code paths. Currently the seed services are deployed, but not tested. A future change could test discovery and provisioning of an overcloud host using the seed. Change-Id: Iefe16215af6f1a9fb8ec78094cd6bd76a82a040a Story: 2001655 Task: 6683 --- dev/config.sh | 6 ++ dev/functions | 48 +++++++---- kayobe/ansible.py | 7 +- playbooks/kayobe-overcloud-base/pre.yml | 9 +- .../kayobe-seed-base/bifrost-overrides.yml.j2 | 7 ++ playbooks/kayobe-seed-base/overrides.yml.j2 | 14 ++++ playbooks/kayobe-seed-base/post.yml | 7 ++ playbooks/kayobe-seed-base/pre.yml | 84 +++++++++++++++++++ playbooks/kayobe-seed-base/run.yml | 13 +++ roles/kayobe-diagnostics/files/get_logs.sh | 9 +- roles/kayobe-diagnostics/tasks/post.yml | 2 +- zuul.d/jobs.yaml | 24 ++++++ zuul.d/project.yaml | 2 + 13 files changed, 202 insertions(+), 30 deletions(-) create mode 100644 playbooks/kayobe-seed-base/bifrost-overrides.yml.j2 create mode 100644 playbooks/kayobe-seed-base/overrides.yml.j2 create mode 100644 playbooks/kayobe-seed-base/post.yml create mode 100644 playbooks/kayobe-seed-base/pre.yml create mode 100644 playbooks/kayobe-seed-base/run.yml diff --git a/dev/config.sh b/dev/config.sh index 1d4efcad9..8c781199e 100644 --- a/dev/config.sh +++ b/dev/config.sh @@ -10,6 +10,9 @@ # Path to the kayobe virtual environment. #export KAYOBE_VENV_PATH=~/kayobe-venv +# Whether to provision a VM for the seed host. +#export KAYOBE_SEED_VM_PROVISION=1 + # Whether to build container images for the seed services. If 0, they will be # pulled. #export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=0 @@ -17,3 +20,6 @@ # Whether to build container images for the overcloud services. If 0, they will # be pulled. #export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=0 + +# Additional arguments to pass to kayobe commands. +#export KAYOBE_EXTRA_ARGS= diff --git a/dev/functions b/dev/functions index bf0cb6dba..f5264c50e 100644 --- a/dev/functions +++ b/dev/functions @@ -27,6 +27,9 @@ function config_defaults { # Path to the kayobe virtual environment. export KAYOBE_VENV_PATH="${KAYOBE_VENV_PATH:-${HOME}/kayobe-venv}" + # Whether to provision a VM for the seed host. + export KAYOBE_SEED_VM_PROVISION=${KAYOBE_SEED_VM_PROVISION:-1} + # Whether to build container images for the seed services. If 0, they will # be pulled. export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=${KAYOBE_SEED_CONTAINER_IMAGE_BUILD:-0} @@ -34,6 +37,9 @@ function config_defaults { # Whether to build container images for the overcloud services. If 0, they # will be pulled. export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD:-0} + + # Additional arguments to pass to kayobe commands. + export KAYOBE_EXTRA_ARGS=${KAYOBE_EXTRA_ARGS:-} } function config_set { @@ -115,15 +121,21 @@ function environment_setup { cd "${KAYOBE_SOURCE_PATH}" } +function run_kayobe { + # Run a kayobe command, including extra arguments provided via + # $KAYOBE_EXTRA_ARGS. + kayobe ${KAYOBE_EXTRA_ARGS} $* +} + function seed_hypervisor_deploy { # Deploy a seed hypervisor. environment_setup echo "Bootstrapping the ansible control host" - kayobe control host bootstrap + run_kayobe control host bootstrap echo "Configuring the seed hypervisor" - kayobe seed hypervisor host configure + run_kayobe seed hypervisor host configure } function seed_deploy { @@ -131,34 +143,36 @@ function seed_deploy { environment_setup echo "Bootstrapping the ansible control host" - kayobe control host bootstrap + run_kayobe control host bootstrap - echo "Provisioning the seed VM" - kayobe seed vm provision + if [[ ${KAYOBE_SEED_VM_PROVISION} = 1 ]]; then + echo "Provisioning the seed VM" + run_kayobe seed vm provision + fi echo "Configuring the seed host" - kayobe seed host configure + run_kayobe seed host configure # Note: This must currently be before host configure, because host # configure runs kolla-ansible.yml, which validates the presence of the # built deploy images. if is_deploy_image_built_locally; then echo "Building seed deployment images" - kayobe seed deployment image build + run_kayobe seed deployment image build else echo "Not building seed deployment images" fi if [[ ${KAYOBE_SEED_CONTAINER_IMAGE_BUILD} = 1 ]]; then echo "Building seed container images" - kayobe seed container image build + run_kayobe seed container image build else echo "Not pulling seed container images - no such command yet" - #kayobe seed container image pull + #run_kayobe seed container image pull fi echo "Deploying containerised seed services" - kayobe seed service deploy + run_kayobe seed service deploy } function overcloud_deploy { @@ -169,35 +183,35 @@ function overcloud_deploy { environment_setup echo "Bootstrapping the ansible control host" - kayobe control host bootstrap + run_kayobe control host bootstrap echo "Configuring the controller host" - kayobe overcloud host configure + run_kayobe overcloud host configure # Note: This must currently be before host configure, because host # configure runs kolla-ansible.yml, which validates the presence of the # built deploy images. if is_deploy_image_built_locally; then echo "Building overcloud deployment images" - kayobe overcloud deployment image build + run_kayobe overcloud deployment image build else echo "Not building overcloud deployment images" fi if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then echo "Building overcloud container images" - kayobe overcloud container image build + run_kayobe overcloud container image build else echo "Pulling overcloud container images" - kayobe overcloud container image pull + run_kayobe overcloud container image pull fi echo "Deploying containerised overcloud services" - kayobe overcloud service deploy + run_kayobe overcloud service deploy echo "Performing post-deployment configuration" source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh" - kayobe overcloud post configure + run_kayobe overcloud post configure echo "Control plane deployment complete" } diff --git a/kayobe/ansible.py b/kayobe/ansible.py index 3340ca959..db081ffbf 100644 --- a/kayobe/ansible.py +++ b/kayobe/ansible.py @@ -99,7 +99,10 @@ def _validate_args(parsed_args, playbooks): def _get_vars_files(config_path): - """Return a list of Kayobe Ansible configuration variable files.""" + """Return a list of Kayobe Ansible configuration variable files. + + The files will be sorted alphabetically by name. + """ vars_files = [] for vars_file in os.listdir(config_path): abs_path = os.path.join(config_path, vars_file) @@ -107,7 +110,7 @@ def _get_vars_files(config_path): root, ext = os.path.splitext(vars_file) if ext in (".yml", ".yaml", ".json"): vars_files.append(abs_path) - return vars_files + return sorted(vars_files) def build_args(parsed_args, playbooks, diff --git a/playbooks/kayobe-overcloud-base/pre.yml b/playbooks/kayobe-overcloud-base/pre.yml index fdc39e023..1f167b946 100644 --- a/playbooks/kayobe-overcloud-base/pre.yml +++ b/playbooks/kayobe-overcloud-base/pre.yml @@ -3,6 +3,7 @@ vars: logs_dir: "/tmp/logs" kayobe_src_dir: "{{ zuul.project.src_dir }}" + kayobe_config_src_dir: "{{ kayobe_src_dir }}/config/src/kayobe-config" roles: - role: kayobe-diagnostics kayobe_diagnostics_phase: "pre" @@ -31,18 +32,20 @@ - name: Ensure kayobe-config directory exists file: - path: "{{ kayobe_src_dir }}/config/src" + path: "{{ kayobe_config_src_dir }}" state: directory - name: Ensure kayobe-config repository is cloned git: repo: https://github.com/stackhpc/dev-kayobe-config - dest: "{{ kayobe_src_dir }}/config/src/kayobe-config" + dest: "{{ kayobe_config_src_dir }}" + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes + # precedence over the standard config files. - name: Ensure kayobe-config override config file exists template: src: overrides.yml.j2 - dest: "{{ kayobe_src_dir }}/config/src/kayobe-config/etc/kayobe/overrides.yml" + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml" # NOTE(mgoddard): The kayobe dev config by default expects a bridge - # breth1 - to exist with an IP address of 192.168.33.3. diff --git a/playbooks/kayobe-seed-base/bifrost-overrides.yml.j2 b/playbooks/kayobe-seed-base/bifrost-overrides.yml.j2 new file mode 100644 index 000000000..0ada56a2f --- /dev/null +++ b/playbooks/kayobe-seed-base/bifrost-overrides.yml.j2 @@ -0,0 +1,7 @@ +--- +# Don't build an IPA deployment image, instead download upstream images. +create_ipa_image: false +download_ipa: true + +# Don't build a disk image. It takes time and can be unreliable. +use_cirros: true diff --git a/playbooks/kayobe-seed-base/overrides.yml.j2 b/playbooks/kayobe-seed-base/overrides.yml.j2 new file mode 100644 index 000000000..92b5ca7f2 --- /dev/null +++ b/playbooks/kayobe-seed-base/overrides.yml.j2 @@ -0,0 +1,14 @@ +--- +# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as +# Ansible is run directly on the controller. +disable_selinux_do_reboot: false + +# NOTE(mgoddard): We're using a cirros image, which doesn't require the +# resolv.conf work around used for CentOS. +overcloud_host_image_workaround_resolv_enabled: false + +# NOTE(mgoddard): Use a loopback-mounted LVM volume for docker storage since +# the overlay driver doesn't work with the ansible template module until +# ansible 2.4.0, and this is required by bifrost. +seed_lvm_group_data_disks: + - /dev/loop0 diff --git a/playbooks/kayobe-seed-base/post.yml b/playbooks/kayobe-seed-base/post.yml new file mode 100644 index 000000000..5de02240a --- /dev/null +++ b/playbooks/kayobe-seed-base/post.yml @@ -0,0 +1,7 @@ +--- +- hosts: all + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "post" + kayobe_diagnostics_log_dir: "/tmp/logs" + kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" diff --git a/playbooks/kayobe-seed-base/pre.yml b/playbooks/kayobe-seed-base/pre.yml new file mode 100644 index 000000000..f89906ae7 --- /dev/null +++ b/playbooks/kayobe-seed-base/pre.yml @@ -0,0 +1,84 @@ +--- +- hosts: primary + vars: + logs_dir: "/tmp/logs" + kayobe_src_dir: "{{ zuul.project.src_dir }}" + kayobe_config_src_dir: "{{ kayobe_src_dir }}/config/src/kayobe-config" + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "pre" + kayobe_diagnostics_log_dir: "{{ logs_dir }}" + tasks: + - name: Install dbus for debian system + apt: + name: dbus + when: + - ansible_os_family == 'Debian' + become: true + + - block: + # NOTE(mgoddard): The CentOS image used in CI has epel-release installed, + # but the configure-mirrors role used by Zuul disables epel. Since we + # install epel-release and expect epel to be enabled, enable it here. + - name: Ensure yum-utils is installed + yum: + name: yum-utils + state: installed + + - name: Enable the EPEL yum repository + command: yum-config-manager --enable epel + when: ansible_os_family == 'RedHat' + become: true + + - name: Ensure kayobe-config directory exists + file: + path: "{{ kayobe_config_src_dir }}" + state: directory + + - name: Ensure kayobe-config repository is cloned + git: + repo: https://github.com/stackhpc/dev-kayobe-config + dest: "{{ kayobe_config_src_dir }}" + + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes + # precedence over the standard config files. + - name: Ensure kayobe-config override config file exists + template: + src: overrides.yml.j2 + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml" + + - name: Ensure bifrost overrides directory exists + file: + path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost" + state: "directory" + + - name: Ensure bifrost overrides file exists + template: + src: bifrost-overrides.yml.j2 + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/bifrost.yml" + + # NOTE(mgoddard): Create a loopback device backed by a file for docker + # storage. We do this since the overlay driver doesn't work with the + # ansible template module until ansible 2.4.0, and this is required by + # bifrost. + - name: Ensure a docker storage backing file exists + command: truncate -s 20G /tmp/docker-storage + + - name: Ensure the docker storage loopback device is created + command: losetup /dev/loop0 /tmp/docker-storage + become: true + + # NOTE(mgoddard): The kayobe dev config by default expects a bridge - + # breth1 - to exist on the seed with an IP address of 192.168.33.5. + - name: Ensure all-in-one network bridge interface exists + command: "{{ item }}" + become: true + with_items: + - "ip l add breth1 type bridge" + - "ip l set breth1 up" + - "ip a add 192.168.33.5/24 dev breth1" + + - name: Ensure kayobe is installed + shell: + cmd: dev/install.sh > {{ logs_dir }}/ansible/install + chdir: "{{ kayobe_src_dir }}" diff --git a/playbooks/kayobe-seed-base/run.yml b/playbooks/kayobe-seed-base/run.yml new file mode 100644 index 000000000..99b6851f2 --- /dev/null +++ b/playbooks/kayobe-seed-base/run.yml @@ -0,0 +1,13 @@ +--- +- hosts: primary + vars: + kayobe_src_dir: "{{ zuul.project.src_dir }}" + logs_dir: "/tmp/logs" + tasks: + - name: Ensure seed is deployed + shell: + cmd: dev/seed-deploy.sh > {{ logs_dir }}/ansible/seed-deploy + chdir: "{{ kayobe_src_dir }}" + environment: + # Don't provision a seed VM - use the Zuul VM as the seed host. + KAYOBE_SEED_VM_PROVISION: 0 diff --git a/roles/kayobe-diagnostics/files/get_logs.sh b/roles/kayobe-diagnostics/files/get_logs.sh index 2a08f2907..9408b012c 100644 --- a/roles/kayobe-diagnostics/files/get_logs.sh +++ b/roles/kayobe-diagnostics/files/get_logs.sh @@ -28,6 +28,8 @@ copy_logs() { parted -l > ${LOG_DIR}/system_logs/parted-l.txt mount > ${LOG_DIR}/system_logs/mount.txt env > ${LOG_DIR}/system_logs/env.txt + ip address > ${LOG_DIR}/system_logs/ip-address.txt + ip route > ${LOG_DIR}/system_logs/ip-route.txt if [ `command -v dpkg` ]; then dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt @@ -50,13 +52,6 @@ copy_logs() { # logs.openstack.org clicking results in the browser shows the # files, rather than trying to send it to another app or make you # download it, etc. - - # Rename files to .txt; this is so that when displayed via - # logs.openstack.org clicking results in the browser shows the - # files, rather than trying to send it to another app or make you - # download it, etc. - - # Rename all .log files to .txt files for f in $(find ${LOG_DIR}/{system_logs,kolla,docker_logs} -name "*.log"); do mv $f ${f/.log/.txt} done diff --git a/roles/kayobe-diagnostics/tasks/post.yml b/roles/kayobe-diagnostics/tasks/post.yml index cc42f13d1..395e135f3 100644 --- a/roles/kayobe-diagnostics/tasks/post.yml +++ b/roles/kayobe-diagnostics/tasks/post.yml @@ -16,6 +16,6 @@ - name: Download logs to executor synchronize: - src: "{{ kayobe_diagnostics_log_dir }}" + src: "{{ kayobe_diagnostics_log_dir }}/" dest: "{{ kayobe_diagnostics_executor_log_dir }}/" mode: pull diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index b5a91e0ed..36af41d54 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -16,6 +16,10 @@ # Base job for testing overcloud deployment. - job: name: kayobe-overcloud-base + description: | + Base job for testing overcloud deployment. + + Configures the primary VM as an overcloud controller. pre-run: playbooks/kayobe-overcloud-base/pre.yml run: playbooks/kayobe-overcloud-base/run.yml post-run: playbooks/kayobe-overcloud-base/post.yml @@ -29,3 +33,23 @@ name: kayobe-overcloud-centos parent: kayobe-overcloud-base nodeset: kayobe-centos + +- job: + name: kayobe-seed-base + description: | + Base job for testing seed deployment. + + Configures the primary VM as a seed. + pre-run: playbooks/kayobe-seed-base/pre.yml + run: playbooks/kayobe-seed-base/run.yml + post-run: playbooks/kayobe-seed-base/post.yml + attempts: 1 + timeout: 5400 + irrelevant-files: + - ^.*\.rst$ + - ^doc/.* + +- job: + name: kayobe-seed-centos + parent: kayobe-seed-base + nodeset: kayobe-centos diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 6fb1b3ccb..b1a516423 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -10,6 +10,7 @@ - kayobe-tox-ansible-syntax - kayobe-tox-ansible - kayobe-overcloud-centos + - kayobe-seed-centos gate: queue: kayobe @@ -22,3 +23,4 @@ - kayobe-tox-ansible-syntax - kayobe-tox-ansible - kayobe-overcloud-centos + - kayobe-seed-centos