From b4bc89dfff7b0b22514256aa0f8cabe0eb96139a Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Wed, 7 Nov 2018 15:24:12 +0000
Subject: [PATCH] Add a zuul job to deploy and teardown a tenks cluster

This stops short of enrolling the nodes in Ironic, but provides
some coverage of the earlier stages.

Change-Id: I48a0bafaff1555caea0a7d216ac6f6a4f11c2b55
Depends-On: https://review.openstack.org/#/c/615544/
Depends-On: https://review.openstack.org/#/c/616510/
Depends-On: https://review.openstack.org/#/c/616501/
---
 ansible/deploy.yml                            |  2 +
 ansible/teardown.yml                          |  2 +
 playbooks/tenks-deploy-teardown/post.yml      | 13 +++++
 playbooks/tenks-deploy-teardown/pre.yml       | 40 +++++++++++++
 playbooks/tenks-deploy-teardown/run.yml       | 32 +++++++++++
 .../templates/tenks-overrides.yml.j2          | 31 ++++++++++
 .../tenks-deploy-teardown/vars/common.yml     | 10 ++++
 roles/tenks-ci-prep/tasks/main.yml            | 14 +++++
 roles/tenks-diagnostics/README.rst            |  6 ++
 roles/tenks-diagnostics/defaults/main.yml     | 13 +++++
 .../files/dump_libvirt_logs.sh                | 26 +++++++++
 roles/tenks-diagnostics/files/get_logs.sh     | 57 +++++++++++++++++++
 roles/tenks-diagnostics/tasks/created.yml     | 14 +++++
 roles/tenks-diagnostics/tasks/main.yml        |  2 +
 roles/tenks-diagnostics/tasks/post.yml        | 24 ++++++++
 roles/tenks-diagnostics/tasks/pre.yml         | 16 ++++++
 zuul.d/base.yaml                              | 21 +++++++
 zuul.d/jobs.yaml                              | 10 ++++
 zuul.d/project.yaml                           |  4 ++
 19 files changed, 337 insertions(+)
 create mode 100644 playbooks/tenks-deploy-teardown/post.yml
 create mode 100644 playbooks/tenks-deploy-teardown/pre.yml
 create mode 100644 playbooks/tenks-deploy-teardown/run.yml
 create mode 100644 playbooks/tenks-deploy-teardown/templates/tenks-overrides.yml.j2
 create mode 100644 playbooks/tenks-deploy-teardown/vars/common.yml
 create mode 100644 roles/tenks-ci-prep/tasks/main.yml
 create mode 100644 roles/tenks-diagnostics/README.rst
 create mode 100644 roles/tenks-diagnostics/defaults/main.yml
 create mode 100644 roles/tenks-diagnostics/files/dump_libvirt_logs.sh
 create mode 100644 roles/tenks-diagnostics/files/get_logs.sh
 create mode 100644 roles/tenks-diagnostics/tasks/created.yml
 create mode 100644 roles/tenks-diagnostics/tasks/main.yml
 create mode 100644 roles/tenks-diagnostics/tasks/post.yml
 create mode 100644 roles/tenks-diagnostics/tasks/pre.yml
 create mode 100644 zuul.d/base.yaml

diff --git a/ansible/deploy.yml b/ansible/deploy.yml
index 94aa32f..2a7f0ad 100644
--- a/ansible/deploy.yml
+++ b/ansible/deploy.yml
@@ -22,9 +22,11 @@
 
 - name: Perform Ironic node configuration
   import_playbook: node_enrolment.yml
+  tags: openstack
 
 - name: Register flavors in Nova
   import_playbook: flavor_registration.yml
+  tags: openstack
 
 - name: Clean up Tenks state
   import_playbook: cleanup_state.yml
diff --git a/ansible/teardown.yml b/ansible/teardown.yml
index 346394f..a1b6470 100644
--- a/ansible/teardown.yml
+++ b/ansible/teardown.yml
@@ -10,9 +10,11 @@
 
 - name: Deregister flavors in Nova
   import_playbook: flavor_registration.yml
+  tags: openstack
 
 - name: Perform Ironic node deconfiguration
   import_playbook: node_enrolment.yml
+  tags: openstack
 
 - name: Deconfigure virtual node BMCs
   import_playbook: node_bmc.yml
diff --git a/playbooks/tenks-deploy-teardown/post.yml b/playbooks/tenks-deploy-teardown/post.yml
new file mode 100644
index 0000000..7ba311e
--- /dev/null
+++ b/playbooks/tenks-deploy-teardown/post.yml
@@ -0,0 +1,13 @@
+---
+- name: tenks-deploy-teardown post run
+  hosts: all
+  pre_tasks:
+    - name: Include common variables
+      include_vars: common.yml
+  roles:
+    - role: tenks-diagnostics
+      tenks_diagnostics_phase: "post"
+      tenks_diagnostics_log_dir: "{{ logs_dir }}"
+      tenks_diagnostics_config_dir: "{{ config_dir }}"
+      tenks_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+
diff --git a/playbooks/tenks-deploy-teardown/pre.yml b/playbooks/tenks-deploy-teardown/pre.yml
new file mode 100644
index 0000000..f3debd3
--- /dev/null
+++ b/playbooks/tenks-deploy-teardown/pre.yml
@@ -0,0 +1,40 @@
+---
+- hosts: all
+  pre_tasks:
+    - name: Include common variables
+      include_vars: common.yml
+  roles:
+    - role: tenks-diagnostics
+      tenks_diagnostics_phase: "pre"
+      tenks_diagnostics_log_dir: "{{ logs_dir }}"
+    - role: tenks-ci-prep
+  tasks:
+    - name: Ensure config directory exists
+      file:
+        path: "{{ config_dir }}"
+        state: "directory"
+
+    - name: Template tenks overrides
+      template:
+        src: tenks-overrides.yml.j2
+        dest: "{{ tenks_overrides_path }}"
+
+    - name: Create virtualenv for tenks
+      pip:
+        requirements: "{{ tenks_src_dir }}/requirements.txt"
+        virtualenv: "{{ tenks_venv }}"
+
+    - name: Install Ansible Galaxy dependencies
+      vars:
+        ansible_galaxy_bin: "{{ tenks_venv }}/bin/ansible-galaxy"
+      command: >-
+        {{ ansible_galaxy_bin }} install -r {{ tenks_src_dir }}/requirements.yml
+        -p {{ tenks_src_dir }}/ansible/roles
+      chdir: "{{ tenks_src_dir }}"
+
+    - name: Ensure network bridge for physnet1 exists
+      command: "{{ item }}"
+      become: true
+      with_items:
+        - "ip l add breth1 type bridge"
+        - "ip l set breth1 up"
diff --git a/playbooks/tenks-deploy-teardown/run.yml b/playbooks/tenks-deploy-teardown/run.yml
new file mode 100644
index 0000000..c8dff04
--- /dev/null
+++ b/playbooks/tenks-deploy-teardown/run.yml
@@ -0,0 +1,32 @@
+---
+- hosts: all
+  vars:
+    ansible_playbook_bin: "{{ tenks_venv }}/bin/ansible-playbook"
+  tasks:
+    - name: Include common variables
+      include_vars: common.yml
+
+    - name: Deploy tenks cluster
+      shell:
+        cmd: >-
+          {{ ansible_playbook_bin }} -vvv --skip-tags openstack
+          --inventory ansible/inventory
+          --extra-vars=@{{ tenks_overrides_path }}
+          ansible/deploy.yml > {{ logs_dir }}/ansible/tenks-deploy
+        chdir: "{{ tenks_src_dir }}"
+
+    - name: Get some diagnostic info for the tenks cluster
+      include_role:
+        name: tenks-diagnostics
+      vars:
+        tenks_diagnostics_phase: "created"
+        tenks_diagnostics_log_dir: "{{ logs_dir }}"
+
+    - name: Teardown tenks cluster
+      shell:
+        cmd: >-
+          {{ ansible_playbook_bin }} -vvv --skip-tags openstack
+          --inventory ansible/inventory
+          --extra-vars=@{{ tenks_overrides_path }}
+          ansible/teardown.yml > {{ logs_dir }}/ansible/tenks-teardown
+        chdir: "{{ tenks_src_dir }}"
diff --git a/playbooks/tenks-deploy-teardown/templates/tenks-overrides.yml.j2 b/playbooks/tenks-deploy-teardown/templates/tenks-overrides.yml.j2
new file mode 100644
index 0000000..89dba73
--- /dev/null
+++ b/playbooks/tenks-deploy-teardown/templates/tenks-overrides.yml.j2
@@ -0,0 +1,31 @@
+---
+# This file holds the config given to Tenks when running the zuul job,
+# tenks-deploy-teardown. It assumes the existence of the bridge `breth1`.
+
+node_types:
+  type0:
+    memory_mb: 1024
+    vcpus: 1
+    volumes:
+      # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent:
+      # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290
+      - capacity: 4GiB
+    physical_networks:
+      - physnet1
+    console_log_enabled: true
+
+specs:
+  - type: type0
+    count: 2
+    ironic_config:
+      resource_class: test-rc
+      network_interface: flat
+
+nova_flavors:
+  - resource_class: test-rc
+    node_type: type0
+
+physnet_mappings:
+  physnet1: breth1
+
+python_upper_constraints_url: file://{{ upper_constraints_path }}
diff --git a/playbooks/tenks-deploy-teardown/vars/common.yml b/playbooks/tenks-deploy-teardown/vars/common.yml
new file mode 100644
index 0000000..150c498
--- /dev/null
+++ b/playbooks/tenks-deploy-teardown/vars/common.yml
@@ -0,0 +1,10 @@
+---
+
+# Variables shared between the playbooks
+
+tenks_src_dir: "{{ ansible_env.HOME ~ '/' ~ zuul.projects['git.openstack.org/openstack/tenks'].src_dir }}"
+upper_constraints_path: "{{ ansible_env.HOME ~ '/' ~ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}"
+tenks_venv: "{{ ansible_env.HOME ~ '/' ~ 'venv-tenks' }}"
+config_dir: "{{ ansible_env.HOME ~ '/' ~ 'tenks-config' }}"
+tenks_overrides_path: "{{ config_dir ~ '/' ~ 'tenks-overrides.yml' }}"
+logs_dir: "/tmp/logs"
diff --git a/roles/tenks-ci-prep/tasks/main.yml b/roles/tenks-ci-prep/tasks/main.yml
new file mode 100644
index 0000000..845124d
--- /dev/null
+++ b/roles/tenks-ci-prep/tasks/main.yml
@@ -0,0 +1,14 @@
+---
+- block:
+    # NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
+    # but the configure-mirrors role used by Zuul disables epel. Since we
+    # install epel-release and expect epel to be enabled, enable it here.
+    - name: Ensure yum-utils is installed
+      yum:
+        name: yum-utils
+        state: installed
+
+    - name: Enable the EPEL yum repository
+      command: yum-config-manager --enable epel
+  when: ansible_os_family == 'RedHat'
+  become: true
diff --git a/roles/tenks-diagnostics/README.rst b/roles/tenks-diagnostics/README.rst
new file mode 100644
index 0000000..0928938
--- /dev/null
+++ b/roles/tenks-diagnostics/README.rst
@@ -0,0 +1,6 @@
+==================
+Tenks Diagnostics
+==================
+
+Ansible role to collect diagnostic information following a CI job performing
+integration testing.
diff --git a/roles/tenks-diagnostics/defaults/main.yml b/roles/tenks-diagnostics/defaults/main.yml
new file mode 100644
index 0000000..0197c8b
--- /dev/null
+++ b/roles/tenks-diagnostics/defaults/main.yml
@@ -0,0 +1,13 @@
+---
+# Job phase - one of 'pre', 'created,  or 'post'. The created phase is
+# designed to called when the virtual machines have been created.
+tenks_diagnostics_phase:
+
+# Directory on the remote host in which to save diagnostics.
+tenks_diagnostics_logs_dir:
+
+# Directory on the remote host in which configuration is stored.
+tenks_diagnostics_config_dir:
+
+# Directory on the executor in which to save logs.
+tenks_diagnostics_executor_log_dir:
diff --git a/roles/tenks-diagnostics/files/dump_libvirt_logs.sh b/roles/tenks-diagnostics/files/dump_libvirt_logs.sh
new file mode 100644
index 0000000..d319671
--- /dev/null
+++ b/roles/tenks-diagnostics/files/dump_libvirt_logs.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# This script will query libvirt to get some information useful for
+# debugging
+
+# Environment variables:
+# $LOG_DIR is the directory to copy logs to.
+# $CONFIG_DIR is the directory to copy configuration from.
+
+set +o errexit
+
+copy_logs() {
+    if ! command -v virsh > /dev/null 2>&1; then
+        return 0
+    fi
+
+    virsh list --all > ${LOG_DIR}/libvirt_logs/list.txt
+    virsh list --all --name | while read vm; do
+        if [ "$vm" != "" ]; then
+            virsh dumpxml "$vm" > ${LOG_DIR}/libvirt_logs/"$vm".txt
+        fi
+    done
+
+}
+
+copy_logs
diff --git a/roles/tenks-diagnostics/files/get_logs.sh b/roles/tenks-diagnostics/files/get_logs.sh
new file mode 100644
index 0000000..0858673
--- /dev/null
+++ b/roles/tenks-diagnostics/files/get_logs.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# NOTE(wszumski): This has been adapted from tests/get_logs.sh in Kayobe.
+
+# Environment variables:
+# $LOG_DIR is the directory to copy logs to.
+# $CONFIG_DIR is the directory to copy configuration from.
+
+set +o errexit
+
+copy_logs() {
+    if [[ -d ${CONFIG_DIR} ]]; then
+        cp -rnL ${CONFIG_DIR}/* ${LOG_DIR}/config/
+    fi
+    cp -rvnL /var/log/* ${LOG_DIR}/system_logs/
+
+    if [[ -x "$(command -v journalctl)" ]]; then
+        journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.txt
+    fi
+
+    cp -r /etc/sudoers.d ${LOG_DIR}/system_logs/
+    cp /etc/sudoers ${LOG_DIR}/system_logs/sudoers.txt
+
+    df -h > ${LOG_DIR}/system_logs/df.txt
+    free  > ${LOG_DIR}/system_logs/free.txt
+    parted -l > ${LOG_DIR}/system_logs/parted-l.txt
+    mount > ${LOG_DIR}/system_logs/mount.txt
+    env > ${LOG_DIR}/system_logs/env.txt
+    ip address > ${LOG_DIR}/system_logs/ip-address.txt
+    ip route > ${LOG_DIR}/system_logs/ip-route.txt
+
+    iptables-save > ${LOG_DIR}/system_logs/iptables.txt
+
+    if [ `command -v dpkg` ]; then
+        dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt
+    fi
+    if [ `command -v rpm` ]; then
+        rpm -qa > ${LOG_DIR}/system_logs/rpm-qa.txt
+    fi
+
+    # final memory usage and process list
+    ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ${LOG_DIR}/system_logs/ps.txt
+
+    # Rename files to .txt; this is so that when displayed via
+    # logs.openstack.org clicking results in the browser shows the
+    # files, rather than trying to send it to another app or make you
+    # download it, etc.
+    for f in $(find ${LOG_DIR}/{system_logs,libvirt_logs} -name "*.log"); do
+        mv $f ${f/.log/.txt}
+    done
+
+    chmod -R 777 ${LOG_DIR}
+    find ${LOG_DIR}/{system_logs,libvirt_logs} -iname '*.txt' -execdir gzip -f -9 {} \+
+    find ${LOG_DIR}/{system_logs,libvirt_logs} -iname '*.json' -execdir gzip -f -9 {} \+
+}
+
+copy_logs
diff --git a/roles/tenks-diagnostics/tasks/created.yml b/roles/tenks-diagnostics/tasks/created.yml
new file mode 100644
index 0000000..10032a8
--- /dev/null
+++ b/roles/tenks-diagnostics/tasks/created.yml
@@ -0,0 +1,14 @@
+---
+
+- name: Run diagnostics script
+  script: dump_libvirt_logs.sh
+  register: dump_libvirt_logs_result
+  become: true
+  failed_when: false
+  environment:
+    LOG_DIR: "{{ tenks_diagnostics_log_dir }}"
+    CONFIG_DIR: "{{ tenks_diagnostics_config_dir }}"
+
+- name: Print dump_libvirt_logs.sh output
+  debug:
+    msg: "{{ dump_libvirt_logs_result.stdout }}"
diff --git a/roles/tenks-diagnostics/tasks/main.yml b/roles/tenks-diagnostics/tasks/main.yml
new file mode 100644
index 0000000..4dc1719
--- /dev/null
+++ b/roles/tenks-diagnostics/tasks/main.yml
@@ -0,0 +1,2 @@
+---
+- include: "{{ tenks_diagnostics_phase }}.yml"
diff --git a/roles/tenks-diagnostics/tasks/post.yml b/roles/tenks-diagnostics/tasks/post.yml
new file mode 100644
index 0000000..57500e9
--- /dev/null
+++ b/roles/tenks-diagnostics/tasks/post.yml
@@ -0,0 +1,24 @@
+---
+- name: Write host variables to a file
+  copy:
+    content: "{{ hostvars[inventory_hostname] | to_nice_json }}"
+    dest: "{{ tenks_diagnostics_log_dir }}/facts.json"
+
+- name: Run diagnostics script
+  script: get_logs.sh
+  register: get_logs_result
+  become: true
+  failed_when: false
+  environment:
+    LOG_DIR: "{{ tenks_diagnostics_log_dir }}"
+    CONFIG_DIR: "{{ tenks_diagnostics_config_dir }}"
+
+- name: Print get_logs.sh output
+  debug:
+    msg: "{{ get_logs_result.stdout }}"
+
+- name: Download logs to executor
+  synchronize:
+    src: "{{ tenks_diagnostics_log_dir }}/"
+    dest: "{{ tenks_diagnostics_executor_log_dir }}/"
+    mode: pull
diff --git a/roles/tenks-diagnostics/tasks/pre.yml b/roles/tenks-diagnostics/tasks/pre.yml
new file mode 100644
index 0000000..345655c
--- /dev/null
+++ b/roles/tenks-diagnostics/tasks/pre.yml
@@ -0,0 +1,16 @@
+---
+- name: Ensure node log directory exists
+  file:
+    path: "{{ tenks_diagnostics_log_dir }}"
+    state: "directory"
+
+- name: Ensure node log subdirectories exist
+  file:
+    path: "{{ tenks_diagnostics_log_dir }}/{{ item }}"
+    state: "directory"
+    mode: 0777
+  with_items:
+    - "libvirt_logs"
+    - "config"
+    - "system_logs"
+    - "ansible"
diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml
new file mode 100644
index 0000000..e079844
--- /dev/null
+++ b/zuul.d/base.yaml
@@ -0,0 +1,21 @@
+---
+
+- job:
+    name: tenks-deploy-teardown-base
+    description: |
+      Creates the tenks virtual machines and sets up the virtual BMC daemon.
+      This stops short of enrolling the machines in Ironic. The cluster is
+      then destroyed again.
+    pre-run: playbooks/tenks-deploy-teardown/pre.yml
+    run: playbooks/tenks-deploy-teardown/run.yml
+    post-run: playbooks/tenks-deploy-teardown/post.yml
+    required-projects:
+      - name: openstack/requirements
+    attempts: 1
+    timeout: 1800
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*
+      - ^releasenotes/.*
+      - ^setup.cfg$
+      - ^tox.ini$
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 30ce40c..61c582e 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -7,3 +7,13 @@
     vars:
       tox_envlist: alint
 
+- job:
+    name: tenks-deploy-teardown-centos
+    parent: tenks-deploy-teardown-base
+    nodeset: centos-7
+
+- job:
+    name: tenks-deploy-teardown-ubuntu
+    parent: tenks-deploy-teardown-base
+    nodeset: ubuntu-bionic
+
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 5e9a870..7e9547b 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -9,8 +9,12 @@
       jobs:
         - build-openstack-releasenotes
         - tenks-tox-ansible-lint
+        - tenks-deploy-teardown-centos
+        - tenks-deploy-teardown-ubuntu
     gate:
       queue: tenks
       jobs:
         - build-openstack-releasenotes
         - tenks-tox-ansible-lint
+        - tenks-deploy-teardown-centos
+        - tenks-deploy-teardown-ubuntu