diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 94aa32f..2a7f0ad 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -22,9 +22,11 @@ - name: Perform Ironic node configuration import_playbook: node_enrolment.yml + tags: openstack - name: Register flavors in Nova import_playbook: flavor_registration.yml + tags: openstack - name: Clean up Tenks state import_playbook: cleanup_state.yml diff --git a/ansible/teardown.yml b/ansible/teardown.yml index 346394f..a1b6470 100644 --- a/ansible/teardown.yml +++ b/ansible/teardown.yml @@ -10,9 +10,11 @@ - name: Deregister flavors in Nova import_playbook: flavor_registration.yml + tags: openstack - name: Perform Ironic node deconfiguration import_playbook: node_enrolment.yml + tags: openstack - name: Deconfigure virtual node BMCs import_playbook: node_bmc.yml diff --git a/playbooks/tenks-deploy-teardown/post.yml b/playbooks/tenks-deploy-teardown/post.yml new file mode 100644 index 0000000..7ba311e --- /dev/null +++ b/playbooks/tenks-deploy-teardown/post.yml @@ -0,0 +1,13 @@ +--- +- name: tenks-deploy-teardown post run + hosts: all + pre_tasks: + - name: Include common variables + include_vars: common.yml + roles: + - role: tenks-diagnostics + tenks_diagnostics_phase: "post" + tenks_diagnostics_log_dir: "{{ logs_dir }}" + tenks_diagnostics_config_dir: "{{ config_dir }}" + tenks_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + diff --git a/playbooks/tenks-deploy-teardown/pre.yml b/playbooks/tenks-deploy-teardown/pre.yml new file mode 100644 index 0000000..f3debd3 --- /dev/null +++ b/playbooks/tenks-deploy-teardown/pre.yml @@ -0,0 +1,40 @@ +--- +- hosts: all + pre_tasks: + - name: Include common variables + include_vars: common.yml + roles: + - role: tenks-diagnostics + tenks_diagnostics_phase: "pre" + tenks_diagnostics_log_dir: "{{ logs_dir }}" + - role: tenks-ci-prep + tasks: + - name: Ensure config directory exists + file: + path: "{{ config_dir }}" + state: "directory" + + - name: Template tenks overrides + template: + src: tenks-overrides.yml.j2 + dest: "{{ tenks_overrides_path }}" + + - name: Create virtualenv for tenks + pip: + requirements: "{{ tenks_src_dir }}/requirements.txt" + virtualenv: "{{ tenks_venv }}" + + - name: Install Ansible Galaxy dependencies + vars: + ansible_galaxy_bin: "{{ tenks_venv }}/bin/ansible-galaxy" + command: >- + {{ ansible_galaxy_bin }} install -r {{ tenks_src_dir }}/requirements.yml + -p {{ tenks_src_dir }}/ansible/roles + chdir: "{{ tenks_src_dir }}" + + - name: Ensure network bridge for physnet1 exists + command: "{{ item }}" + become: true + with_items: + - "ip l add breth1 type bridge" + - "ip l set breth1 up" diff --git a/playbooks/tenks-deploy-teardown/run.yml b/playbooks/tenks-deploy-teardown/run.yml new file mode 100644 index 0000000..c8dff04 --- /dev/null +++ b/playbooks/tenks-deploy-teardown/run.yml @@ -0,0 +1,32 @@ +--- +- hosts: all + vars: + ansible_playbook_bin: "{{ tenks_venv }}/bin/ansible-playbook" + tasks: + - name: Include common variables + include_vars: common.yml + + - name: Deploy tenks cluster + shell: + cmd: >- + {{ ansible_playbook_bin }} -vvv --skip-tags openstack + --inventory ansible/inventory + --extra-vars=@{{ tenks_overrides_path }} + ansible/deploy.yml > {{ logs_dir }}/ansible/tenks-deploy + chdir: "{{ tenks_src_dir }}" + + - name: Get some diagnostic info for the tenks cluster + include_role: + name: tenks-diagnostics + vars: + tenks_diagnostics_phase: "created" + tenks_diagnostics_log_dir: "{{ logs_dir }}" + + - name: Teardown tenks cluster + shell: + cmd: >- + {{ ansible_playbook_bin }} -vvv --skip-tags openstack + --inventory ansible/inventory + --extra-vars=@{{ tenks_overrides_path }} + ansible/teardown.yml > {{ logs_dir }}/ansible/tenks-teardown + chdir: "{{ tenks_src_dir }}" diff --git a/playbooks/tenks-deploy-teardown/templates/tenks-overrides.yml.j2 b/playbooks/tenks-deploy-teardown/templates/tenks-overrides.yml.j2 new file mode 100644 index 0000000..89dba73 --- /dev/null +++ b/playbooks/tenks-deploy-teardown/templates/tenks-overrides.yml.j2 @@ -0,0 +1,31 @@ +--- +# This file holds the config given to Tenks when running the zuul job, +# tenks-deploy-teardown. It assumes the existence of the bridge `breth1`. + +node_types: + type0: + memory_mb: 1024 + vcpus: 1 + volumes: + # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent: + # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290 + - capacity: 4GiB + physical_networks: + - physnet1 + console_log_enabled: true + +specs: + - type: type0 + count: 2 + ironic_config: + resource_class: test-rc + network_interface: flat + +nova_flavors: + - resource_class: test-rc + node_type: type0 + +physnet_mappings: + physnet1: breth1 + +python_upper_constraints_url: file://{{ upper_constraints_path }} diff --git a/playbooks/tenks-deploy-teardown/vars/common.yml b/playbooks/tenks-deploy-teardown/vars/common.yml new file mode 100644 index 0000000..150c498 --- /dev/null +++ b/playbooks/tenks-deploy-teardown/vars/common.yml @@ -0,0 +1,10 @@ +--- + +# Variables shared between the playbooks + +tenks_src_dir: "{{ ansible_env.HOME ~ '/' ~ zuul.projects['git.openstack.org/openstack/tenks'].src_dir }}" +upper_constraints_path: "{{ ansible_env.HOME ~ '/' ~ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}" +tenks_venv: "{{ ansible_env.HOME ~ '/' ~ 'venv-tenks' }}" +config_dir: "{{ ansible_env.HOME ~ '/' ~ 'tenks-config' }}" +tenks_overrides_path: "{{ config_dir ~ '/' ~ 'tenks-overrides.yml' }}" +logs_dir: "/tmp/logs" diff --git a/roles/tenks-ci-prep/tasks/main.yml b/roles/tenks-ci-prep/tasks/main.yml new file mode 100644 index 0000000..845124d --- /dev/null +++ b/roles/tenks-ci-prep/tasks/main.yml @@ -0,0 +1,14 @@ +--- +- block: + # NOTE(mgoddard): The CentOS image used in CI has epel-release installed, + # but the configure-mirrors role used by Zuul disables epel. Since we + # install epel-release and expect epel to be enabled, enable it here. + - name: Ensure yum-utils is installed + yum: + name: yum-utils + state: installed + + - name: Enable the EPEL yum repository + command: yum-config-manager --enable epel + when: ansible_os_family == 'RedHat' + become: true diff --git a/roles/tenks-diagnostics/README.rst b/roles/tenks-diagnostics/README.rst new file mode 100644 index 0000000..0928938 --- /dev/null +++ b/roles/tenks-diagnostics/README.rst @@ -0,0 +1,6 @@ +================== +Tenks Diagnostics +================== + +Ansible role to collect diagnostic information following a CI job performing +integration testing. diff --git a/roles/tenks-diagnostics/defaults/main.yml b/roles/tenks-diagnostics/defaults/main.yml new file mode 100644 index 0000000..0197c8b --- /dev/null +++ b/roles/tenks-diagnostics/defaults/main.yml @@ -0,0 +1,13 @@ +--- +# Job phase - one of 'pre', 'created, or 'post'. The created phase is +# designed to called when the virtual machines have been created. +tenks_diagnostics_phase: + +# Directory on the remote host in which to save diagnostics. +tenks_diagnostics_logs_dir: + +# Directory on the remote host in which configuration is stored. +tenks_diagnostics_config_dir: + +# Directory on the executor in which to save logs. +tenks_diagnostics_executor_log_dir: diff --git a/roles/tenks-diagnostics/files/dump_libvirt_logs.sh b/roles/tenks-diagnostics/files/dump_libvirt_logs.sh new file mode 100644 index 0000000..d319671 --- /dev/null +++ b/roles/tenks-diagnostics/files/dump_libvirt_logs.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# This script will query libvirt to get some information useful for +# debugging + +# Environment variables: +# $LOG_DIR is the directory to copy logs to. +# $CONFIG_DIR is the directory to copy configuration from. + +set +o errexit + +copy_logs() { + if ! command -v virsh > /dev/null 2>&1; then + return 0 + fi + + virsh list --all > ${LOG_DIR}/libvirt_logs/list.txt + virsh list --all --name | while read vm; do + if [ "$vm" != "" ]; then + virsh dumpxml "$vm" > ${LOG_DIR}/libvirt_logs/"$vm".txt + fi + done + +} + +copy_logs diff --git a/roles/tenks-diagnostics/files/get_logs.sh b/roles/tenks-diagnostics/files/get_logs.sh new file mode 100644 index 0000000..0858673 --- /dev/null +++ b/roles/tenks-diagnostics/files/get_logs.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# NOTE(wszumski): This has been adapted from tests/get_logs.sh in Kayobe. + +# Environment variables: +# $LOG_DIR is the directory to copy logs to. +# $CONFIG_DIR is the directory to copy configuration from. + +set +o errexit + +copy_logs() { + if [[ -d ${CONFIG_DIR} ]]; then + cp -rnL ${CONFIG_DIR}/* ${LOG_DIR}/config/ + fi + cp -rvnL /var/log/* ${LOG_DIR}/system_logs/ + + if [[ -x "$(command -v journalctl)" ]]; then + journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.txt + fi + + cp -r /etc/sudoers.d ${LOG_DIR}/system_logs/ + cp /etc/sudoers ${LOG_DIR}/system_logs/sudoers.txt + + df -h > ${LOG_DIR}/system_logs/df.txt + free > ${LOG_DIR}/system_logs/free.txt + parted -l > ${LOG_DIR}/system_logs/parted-l.txt + mount > ${LOG_DIR}/system_logs/mount.txt + env > ${LOG_DIR}/system_logs/env.txt + ip address > ${LOG_DIR}/system_logs/ip-address.txt + ip route > ${LOG_DIR}/system_logs/ip-route.txt + + iptables-save > ${LOG_DIR}/system_logs/iptables.txt + + if [ `command -v dpkg` ]; then + dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt + fi + if [ `command -v rpm` ]; then + rpm -qa > ${LOG_DIR}/system_logs/rpm-qa.txt + fi + + # final memory usage and process list + ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ${LOG_DIR}/system_logs/ps.txt + + # Rename files to .txt; this is so that when displayed via + # logs.openstack.org clicking results in the browser shows the + # files, rather than trying to send it to another app or make you + # download it, etc. + for f in $(find ${LOG_DIR}/{system_logs,libvirt_logs} -name "*.log"); do + mv $f ${f/.log/.txt} + done + + chmod -R 777 ${LOG_DIR} + find ${LOG_DIR}/{system_logs,libvirt_logs} -iname '*.txt' -execdir gzip -f -9 {} \+ + find ${LOG_DIR}/{system_logs,libvirt_logs} -iname '*.json' -execdir gzip -f -9 {} \+ +} + +copy_logs diff --git a/roles/tenks-diagnostics/tasks/created.yml b/roles/tenks-diagnostics/tasks/created.yml new file mode 100644 index 0000000..10032a8 --- /dev/null +++ b/roles/tenks-diagnostics/tasks/created.yml @@ -0,0 +1,14 @@ +--- + +- name: Run diagnostics script + script: dump_libvirt_logs.sh + register: dump_libvirt_logs_result + become: true + failed_when: false + environment: + LOG_DIR: "{{ tenks_diagnostics_log_dir }}" + CONFIG_DIR: "{{ tenks_diagnostics_config_dir }}" + +- name: Print dump_libvirt_logs.sh output + debug: + msg: "{{ dump_libvirt_logs_result.stdout }}" diff --git a/roles/tenks-diagnostics/tasks/main.yml b/roles/tenks-diagnostics/tasks/main.yml new file mode 100644 index 0000000..4dc1719 --- /dev/null +++ b/roles/tenks-diagnostics/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include: "{{ tenks_diagnostics_phase }}.yml" diff --git a/roles/tenks-diagnostics/tasks/post.yml b/roles/tenks-diagnostics/tasks/post.yml new file mode 100644 index 0000000..57500e9 --- /dev/null +++ b/roles/tenks-diagnostics/tasks/post.yml @@ -0,0 +1,24 @@ +--- +- name: Write host variables to a file + copy: + content: "{{ hostvars[inventory_hostname] | to_nice_json }}" + dest: "{{ tenks_diagnostics_log_dir }}/facts.json" + +- name: Run diagnostics script + script: get_logs.sh + register: get_logs_result + become: true + failed_when: false + environment: + LOG_DIR: "{{ tenks_diagnostics_log_dir }}" + CONFIG_DIR: "{{ tenks_diagnostics_config_dir }}" + +- name: Print get_logs.sh output + debug: + msg: "{{ get_logs_result.stdout }}" + +- name: Download logs to executor + synchronize: + src: "{{ tenks_diagnostics_log_dir }}/" + dest: "{{ tenks_diagnostics_executor_log_dir }}/" + mode: pull diff --git a/roles/tenks-diagnostics/tasks/pre.yml b/roles/tenks-diagnostics/tasks/pre.yml new file mode 100644 index 0000000..345655c --- /dev/null +++ b/roles/tenks-diagnostics/tasks/pre.yml @@ -0,0 +1,16 @@ +--- +- name: Ensure node log directory exists + file: + path: "{{ tenks_diagnostics_log_dir }}" + state: "directory" + +- name: Ensure node log subdirectories exist + file: + path: "{{ tenks_diagnostics_log_dir }}/{{ item }}" + state: "directory" + mode: 0777 + with_items: + - "libvirt_logs" + - "config" + - "system_logs" + - "ansible" diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml new file mode 100644 index 0000000..e079844 --- /dev/null +++ b/zuul.d/base.yaml @@ -0,0 +1,21 @@ +--- + +- job: + name: tenks-deploy-teardown-base + description: | + Creates the tenks virtual machines and sets up the virtual BMC daemon. + This stops short of enrolling the machines in Ironic. The cluster is + then destroyed again. + pre-run: playbooks/tenks-deploy-teardown/pre.yml + run: playbooks/tenks-deploy-teardown/run.yml + post-run: playbooks/tenks-deploy-teardown/post.yml + required-projects: + - name: openstack/requirements + attempts: 1 + timeout: 1800 + irrelevant-files: + - ^.*\.rst$ + - ^doc/.* + - ^releasenotes/.* + - ^setup.cfg$ + - ^tox.ini$ diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 30ce40c..61c582e 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -7,3 +7,13 @@ vars: tox_envlist: alint +- job: + name: tenks-deploy-teardown-centos + parent: tenks-deploy-teardown-base + nodeset: centos-7 + +- job: + name: tenks-deploy-teardown-ubuntu + parent: tenks-deploy-teardown-base + nodeset: ubuntu-bionic + diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 5e9a870..7e9547b 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -9,8 +9,12 @@ jobs: - build-openstack-releasenotes - tenks-tox-ansible-lint + - tenks-deploy-teardown-centos + - tenks-deploy-teardown-ubuntu gate: queue: tenks jobs: - build-openstack-releasenotes - tenks-tox-ansible-lint + - tenks-deploy-teardown-centos + - tenks-deploy-teardown-ubuntu