From 4cdf71f5a117d4f8df84e1771ee9d39605cc527e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 12 Mar 2018 12:35:00 +0000 Subject: [PATCH] Add overcloud deployment job Adds a Zuul job to perform deployment of an OpenStack control plane consisting of a single node. Currently no tests are run against the resulting control plane. Change-Id: I2728421cb5dbd1dc370a3aa7d37d913c90ba38a5 Story: #2001655 Task: #6682 --- playbooks/README.rst | 5 ++ .../kayobe-overcloud-base/overrides.yml.j2 | 4 ++ playbooks/kayobe-overcloud-base/post.yml | 7 ++ playbooks/kayobe-overcloud-base/pre.yml | 60 ++++++++++++++++ playbooks/kayobe-overcloud-base/run.yml | 10 +++ roles/README.rst | 5 ++ roles/kayobe-diagnostics/README.rst | 6 ++ roles/kayobe-diagnostics/defaults/main.yml | 9 +++ roles/kayobe-diagnostics/files/get_logs.sh | 69 +++++++++++++++++++ roles/kayobe-diagnostics/tasks/main.yml | 2 + roles/kayobe-diagnostics/tasks/post.yml | 21 ++++++ roles/kayobe-diagnostics/tasks/pre.yml | 17 +++++ zuul.d/jobs.yaml | 17 +++++ zuul.d/nodesets.yaml | 6 ++ zuul.d/project.yaml | 2 + 15 files changed, 240 insertions(+) create mode 100644 playbooks/README.rst create mode 100644 playbooks/kayobe-overcloud-base/overrides.yml.j2 create mode 100644 playbooks/kayobe-overcloud-base/post.yml create mode 100644 playbooks/kayobe-overcloud-base/pre.yml create mode 100644 playbooks/kayobe-overcloud-base/run.yml create mode 100644 roles/README.rst create mode 100644 roles/kayobe-diagnostics/README.rst create mode 100644 roles/kayobe-diagnostics/defaults/main.yml create mode 100644 roles/kayobe-diagnostics/files/get_logs.sh create mode 100644 roles/kayobe-diagnostics/tasks/main.yml create mode 100644 roles/kayobe-diagnostics/tasks/post.yml create mode 100644 roles/kayobe-diagnostics/tasks/pre.yml create mode 100644 zuul.d/nodesets.yaml diff --git a/playbooks/README.rst b/playbooks/README.rst new file mode 100644 index 000000000..bfb4ea594 --- /dev/null +++ b/playbooks/README.rst @@ -0,0 +1,5 @@ +Zuul Ansible Playbooks +====================== + +This directory contains playbooks for use by Zuul. These playbooks are not used +by kayobe - see ``ansible/``. diff --git a/playbooks/kayobe-overcloud-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-base/overrides.yml.j2 new file mode 100644 index 000000000..db327943d --- /dev/null +++ b/playbooks/kayobe-overcloud-base/overrides.yml.j2 @@ -0,0 +1,4 @@ +--- +# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as +# Ansible is run directly on the controller. +disable_selinux_do_reboot: false diff --git a/playbooks/kayobe-overcloud-base/post.yml b/playbooks/kayobe-overcloud-base/post.yml new file mode 100644 index 000000000..5de02240a --- /dev/null +++ b/playbooks/kayobe-overcloud-base/post.yml @@ -0,0 +1,7 @@ +--- +- hosts: all + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "post" + kayobe_diagnostics_log_dir: "/tmp/logs" + kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" diff --git a/playbooks/kayobe-overcloud-base/pre.yml b/playbooks/kayobe-overcloud-base/pre.yml new file mode 100644 index 000000000..fdc39e023 --- /dev/null +++ b/playbooks/kayobe-overcloud-base/pre.yml @@ -0,0 +1,60 @@ +--- +- hosts: primary + vars: + logs_dir: "/tmp/logs" + kayobe_src_dir: "{{ zuul.project.src_dir }}" + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "pre" + kayobe_diagnostics_log_dir: "{{ logs_dir }}" + tasks: + - name: Install dbus for debian system + apt: + name: dbus + when: + - ansible_os_family == 'Debian' + become: true + + - block: + # NOTE(mgoddard): The CentOS image used in CI has epel-release installed, + # but the configure-mirrors role used by Zuul disables epel. Since we + # install epel-release and expect epel to be enabled, enable it here. + - name: Ensure yum-utils is installed + yum: + name: yum-utils + state: installed + + - name: Enable the EPEL yum repository + command: yum-config-manager --enable epel + when: ansible_os_family == 'RedHat' + become: true + + - name: Ensure kayobe-config directory exists + file: + path: "{{ kayobe_src_dir }}/config/src" + state: directory + + - name: Ensure kayobe-config repository is cloned + git: + repo: https://github.com/stackhpc/dev-kayobe-config + dest: "{{ kayobe_src_dir }}/config/src/kayobe-config" + + - name: Ensure kayobe-config override config file exists + template: + src: overrides.yml.j2 + dest: "{{ kayobe_src_dir }}/config/src/kayobe-config/etc/kayobe/overrides.yml" + + # NOTE(mgoddard): The kayobe dev config by default expects a bridge - + # breth1 - to exist with an IP address of 192.168.33.3. + - name: Ensure all-in-one network bridge interface exists + command: "{{ item }}" + become: true + with_items: + - "ip l add breth1 type bridge" + - "ip l set breth1 up" + - "ip a add 192.168.33.3/24 dev breth1" + + - name: Ensure kayobe is installed + shell: + cmd: dev/install.sh > {{ logs_dir }}/ansible/install + chdir: "{{ kayobe_src_dir }}" diff --git a/playbooks/kayobe-overcloud-base/run.yml b/playbooks/kayobe-overcloud-base/run.yml new file mode 100644 index 000000000..3d09ea500 --- /dev/null +++ b/playbooks/kayobe-overcloud-base/run.yml @@ -0,0 +1,10 @@ +--- +- hosts: primary + vars: + kayobe_src_dir: "{{ zuul.project.src_dir }}" + logs_dir: "/tmp/logs" + tasks: + - name: Ensure overcloud is deployed + shell: + cmd: dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy + chdir: "{{ kayobe_src_dir }}" diff --git a/roles/README.rst b/roles/README.rst new file mode 100644 index 000000000..843d4a5eb --- /dev/null +++ b/roles/README.rst @@ -0,0 +1,5 @@ +Zuul Ansible Roles +================== + +This directory contains roles for use by Zuul. These roles are not used by +kayobe - see ``ansible/roles/``. diff --git a/roles/kayobe-diagnostics/README.rst b/roles/kayobe-diagnostics/README.rst new file mode 100644 index 000000000..40549cc82 --- /dev/null +++ b/roles/kayobe-diagnostics/README.rst @@ -0,0 +1,6 @@ +================== +Kayobe Diagnostics +================== + +Ansible role to collect diagnostic information following a CI job performing +integration testing. diff --git a/roles/kayobe-diagnostics/defaults/main.yml b/roles/kayobe-diagnostics/defaults/main.yml new file mode 100644 index 000000000..659ce2927 --- /dev/null +++ b/roles/kayobe-diagnostics/defaults/main.yml @@ -0,0 +1,9 @@ +--- +# Job phase - one of 'pre' or 'post'. +kayobe_diagnostics_phase: + +# Directory on the remote host in which to save diagnostics. +kayobe_diagnostics_logs_dir: + +# Directory on the executor in which to save logs. +kayobe_diagnostics_executor_log_dir: diff --git a/roles/kayobe-diagnostics/files/get_logs.sh b/roles/kayobe-diagnostics/files/get_logs.sh new file mode 100644 index 000000000..2a08f2907 --- /dev/null +++ b/roles/kayobe-diagnostics/files/get_logs.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# NOTE(mgoddard): This has been adapted from tests/get_logs.sh in Kolla +# Ansible. + +set +o errexit + +copy_logs() { + LOG_DIR=/tmp/logs + + cp -rnL /var/lib/docker/volumes/kolla_logs/_data/* ${LOG_DIR}/kolla/ + # TODO(mgoddard): Copy kayobe config + cp -rvnL /var/log/* ${LOG_DIR}/system_logs/ + + + if [[ -x "$(command -v journalctl)" ]]; then + journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.txt + journalctl --no-pager -u docker.service > ${LOG_DIR}/system_logs/docker.log + else + cp /var/log/upstart/docker.log ${LOG_DIR}/system_logs/docker.log + fi + + cp -r /etc/sudoers.d ${LOG_DIR}/system_logs/ + cp /etc/sudoers ${LOG_DIR}/system_logs/sudoers.txt + + df -h > ${LOG_DIR}/system_logs/df.txt + free > ${LOG_DIR}/system_logs/free.txt + parted -l > ${LOG_DIR}/system_logs/parted-l.txt + mount > ${LOG_DIR}/system_logs/mount.txt + env > ${LOG_DIR}/system_logs/env.txt + + if [ `command -v dpkg` ]; then + dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt + fi + if [ `command -v rpm` ]; then + rpm -qa > ${LOG_DIR}/system_logs/rpm-qa.txt + fi + + # final memory usage and process list + ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ${LOG_DIR}/system_logs/ps.txt + + # docker related information + (docker info && docker images && docker ps -a) > ${LOG_DIR}/system_logs/docker-info.txt + + for container in $(docker ps -a --format "{{.Names}}"); do + docker logs --tail all ${container} > ${LOG_DIR}/docker_logs/${container}.txt + done + + # Rename files to .txt; this is so that when displayed via + # logs.openstack.org clicking results in the browser shows the + # files, rather than trying to send it to another app or make you + # download it, etc. + + # Rename files to .txt; this is so that when displayed via + # logs.openstack.org clicking results in the browser shows the + # files, rather than trying to send it to another app or make you + # download it, etc. + + # Rename all .log files to .txt files + for f in $(find ${LOG_DIR}/{system_logs,kolla,docker_logs} -name "*.log"); do + mv $f ${f/.log/.txt} + done + + chmod -R 777 ${LOG_DIR} + find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.txt' -execdir gzip -f -9 {} \+ + find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.json' -execdir gzip -f -9 {} \+ +} + +copy_logs diff --git a/roles/kayobe-diagnostics/tasks/main.yml b/roles/kayobe-diagnostics/tasks/main.yml new file mode 100644 index 000000000..8cc20231b --- /dev/null +++ b/roles/kayobe-diagnostics/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include: "{{ kayobe_diagnostics_phase }}.yml" diff --git a/roles/kayobe-diagnostics/tasks/post.yml b/roles/kayobe-diagnostics/tasks/post.yml new file mode 100644 index 000000000..cc42f13d1 --- /dev/null +++ b/roles/kayobe-diagnostics/tasks/post.yml @@ -0,0 +1,21 @@ +--- +- name: Write host variables to a file + copy: + content: "{{ hostvars[inventory_hostname] | to_nice_json }}" + dest: "{{ kayobe_diagnostics_log_dir }}/facts.json" + +- name: Run diagnostics script + script: get_logs.sh + register: get_logs_result + become: true + failed_when: false + +- name: Print get_logs.sh output + debug: + msg: "{{ get_logs_result.stdout }}" + +- name: Download logs to executor + synchronize: + src: "{{ kayobe_diagnostics_log_dir }}" + dest: "{{ kayobe_diagnostics_executor_log_dir }}/" + mode: pull diff --git a/roles/kayobe-diagnostics/tasks/pre.yml b/roles/kayobe-diagnostics/tasks/pre.yml new file mode 100644 index 000000000..6d51e6674 --- /dev/null +++ b/roles/kayobe-diagnostics/tasks/pre.yml @@ -0,0 +1,17 @@ +--- +- name: Ensure node log directory exists + file: + path: "{{ kayobe_diagnostics_log_dir }}" + state: "directory" + +- name: Ensure node log subdirectories exist + file: + path: "{{ kayobe_diagnostics_log_dir }}/{{ item }}" + state: "directory" + mode: 0777 + with_items: + - "docker_logs" + - "kolla_configs" + - "system_logs" + - "kolla" + - "ansible" diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 973f80e0e..b5a91e0ed 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -12,3 +12,20 @@ parent: openstack-tox-with-sudo vars: tox_envlist: ansible + +# Base job for testing overcloud deployment. +- job: + name: kayobe-overcloud-base + pre-run: playbooks/kayobe-overcloud-base/pre.yml + run: playbooks/kayobe-overcloud-base/run.yml + post-run: playbooks/kayobe-overcloud-base/post.yml + attempts: 1 + timeout: 5400 + irrelevant-files: + - ^.*\.rst$ + - ^doc/.* + +- job: + name: kayobe-overcloud-centos + parent: kayobe-overcloud-base + nodeset: kayobe-centos diff --git a/zuul.d/nodesets.yaml b/zuul.d/nodesets.yaml new file mode 100644 index 000000000..04067f119 --- /dev/null +++ b/zuul.d/nodesets.yaml @@ -0,0 +1,6 @@ +--- +- nodeset: + name: kayobe-centos + nodes: + - name: primary + label: centos-7 diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 905258dba..39e17448a 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -8,6 +8,7 @@ - build-openstack-sphinx-docs - kayobe-tox-ansible-syntax - kayobe-tox-ansible + - kayobe-overcloud-centos gate: queue: kayobe @@ -18,3 +19,4 @@ - build-openstack-sphinx-docs - kayobe-tox-ansible-syntax - kayobe-tox-ansible + - kayobe-overcloud-centos