Merge "Add a zuul job to deploy and teardown a tenks cluster"

ansible/deploy.yml

@@ -22,9 +22,11 @@
 
 - name: Perform Ironic node configuration
   import_playbook: node_enrolment.yml
+  tags: openstack
 
 - name: Register flavors in Nova
   import_playbook: flavor_registration.yml
+  tags: openstack
 
 - name: Clean up Tenks state
   import_playbook: cleanup_state.yml

ansible/teardown.yml

@@ -10,9 +10,11 @@
 
 - name: Deregister flavors in Nova
   import_playbook: flavor_registration.yml
+  tags: openstack
 
 - name: Perform Ironic node deconfiguration
   import_playbook: node_enrolment.yml
+  tags: openstack
 
 - name: Deconfigure virtual node BMCs
   import_playbook: node_bmc.yml

playbooks/tenks-deploy-teardown/post.yml

@@ -0,0 +1,13 @@
+---
+- name: tenks-deploy-teardown post run
+  hosts: all
+  pre_tasks:
+    - name: Include common variables
+      include_vars: common.yml
+  roles:
+    - role: tenks-diagnostics
+      tenks_diagnostics_phase: "post"
+      tenks_diagnostics_log_dir: "{{ logs_dir }}"
+      tenks_diagnostics_config_dir: "{{ config_dir }}"
+      tenks_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+

playbooks/tenks-deploy-teardown/pre.yml

@@ -0,0 +1,40 @@
+---
+- hosts: all
+  pre_tasks:
+    - name: Include common variables
+      include_vars: common.yml
+  roles:
+    - role: tenks-diagnostics
+      tenks_diagnostics_phase: "pre"
+      tenks_diagnostics_log_dir: "{{ logs_dir }}"
+    - role: tenks-ci-prep
+  tasks:
+    - name: Ensure config directory exists
+      file:
+        path: "{{ config_dir }}"
+        state: "directory"
+
+    - name: Template tenks overrides
+      template:
+        src: tenks-overrides.yml.j2
+        dest: "{{ tenks_overrides_path }}"
+
+    - name: Create virtualenv for tenks
+      pip:
+        requirements: "{{ tenks_src_dir }}/requirements.txt"
+        virtualenv: "{{ tenks_venv }}"
+
+    - name: Install Ansible Galaxy dependencies
+      vars:
+        ansible_galaxy_bin: "{{ tenks_venv }}/bin/ansible-galaxy"
+      command: >-
+        {{ ansible_galaxy_bin }} install -r {{ tenks_src_dir }}/requirements.yml
+        -p {{ tenks_src_dir }}/ansible/roles
+      chdir: "{{ tenks_src_dir }}"
+
+    - name: Ensure network bridge for physnet1 exists
+      command: "{{ item }}"
+      become: true
+      with_items:
+        - "ip l add breth1 type bridge"
+        - "ip l set breth1 up"

playbooks/tenks-deploy-teardown/run.yml

@@ -0,0 +1,32 @@
+---
+- hosts: all
+  vars:
+    ansible_playbook_bin: "{{ tenks_venv }}/bin/ansible-playbook"
+  tasks:
+    - name: Include common variables
+      include_vars: common.yml
+
+    - name: Deploy tenks cluster
+      shell:
+        cmd: >-
+          {{ ansible_playbook_bin }} -vvv --skip-tags openstack
+          --inventory ansible/inventory
+          --extra-vars=@{{ tenks_overrides_path }}
+          ansible/deploy.yml > {{ logs_dir }}/ansible/tenks-deploy
+        chdir: "{{ tenks_src_dir }}"
+
+    - name: Get some diagnostic info for the tenks cluster
+      include_role:
+        name: tenks-diagnostics
+      vars:
+        tenks_diagnostics_phase: "created"
+        tenks_diagnostics_log_dir: "{{ logs_dir }}"
+
+    - name: Teardown tenks cluster
+      shell:
+        cmd: >-
+          {{ ansible_playbook_bin }} -vvv --skip-tags openstack
+          --inventory ansible/inventory
+          --extra-vars=@{{ tenks_overrides_path }}
+          ansible/teardown.yml > {{ logs_dir }}/ansible/tenks-teardown
+        chdir: "{{ tenks_src_dir }}"

playbooks/tenks-deploy-teardown/templates/tenks-overrides.yml.j2

@@ -0,0 +1,31 @@
+---
+# This file holds the config given to Tenks when running the zuul job,
+# tenks-deploy-teardown. It assumes the existence of the bridge `breth1`.
+
+node_types:
+  type0:
+    memory_mb: 1024
+    vcpus: 1
+    volumes:
+      # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent:
+      # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290
+      - capacity: 4GiB
+    physical_networks:
+      - physnet1
+    console_log_enabled: true
+
+specs:
+  - type: type0
+    count: 2
+    ironic_config:
+      resource_class: test-rc
+      network_interface: flat
+
+nova_flavors:
+  - resource_class: test-rc
+    node_type: type0
+
+physnet_mappings:
+  physnet1: breth1
+
+python_upper_constraints_url: file://{{ upper_constraints_path }}

playbooks/tenks-deploy-teardown/vars/common.yml

@@ -0,0 +1,10 @@
+---
+
+# Variables shared between the playbooks
+
+tenks_src_dir: "{{ ansible_env.HOME ~ '/' ~ zuul.projects['git.openstack.org/openstack/tenks'].src_dir }}"
+upper_constraints_path: "{{ ansible_env.HOME ~ '/' ~ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}"
+tenks_venv: "{{ ansible_env.HOME ~ '/' ~ 'venv-tenks' }}"
+config_dir: "{{ ansible_env.HOME ~ '/' ~ 'tenks-config' }}"
+tenks_overrides_path: "{{ config_dir ~ '/' ~ 'tenks-overrides.yml' }}"
+logs_dir: "/tmp/logs"

roles/tenks-ci-prep/tasks/main.yml

@@ -0,0 +1,14 @@
+---
+- block:
+    # NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
+    # but the configure-mirrors role used by Zuul disables epel. Since we
+    # install epel-release and expect epel to be enabled, enable it here.
+    - name: Ensure yum-utils is installed
+      yum:
+        name: yum-utils
+        state: installed
+
+    - name: Enable the EPEL yum repository
+      command: yum-config-manager --enable epel
+  when: ansible_os_family == 'RedHat'
+  become: true

roles/tenks-diagnostics/README.rst

@@ -0,0 +1,6 @@
+==================
+Tenks Diagnostics
+==================
+
+Ansible role to collect diagnostic information following a CI job performing
+integration testing.

roles/tenks-diagnostics/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+# Job phase - one of 'pre', 'created,  or 'post'. The created phase is
+# designed to called when the virtual machines have been created.
+tenks_diagnostics_phase:
+
+# Directory on the remote host in which to save diagnostics.
+tenks_diagnostics_logs_dir:
+
+# Directory on the remote host in which configuration is stored.
+tenks_diagnostics_config_dir:
+
+# Directory on the executor in which to save logs.
+tenks_diagnostics_executor_log_dir:

roles/tenks-diagnostics/files/dump_libvirt_logs.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# This script will query libvirt to get some information useful for
+# debugging
+
+# Environment variables:
+# $LOG_DIR is the directory to copy logs to.
+# $CONFIG_DIR is the directory to copy configuration from.
+
+set +o errexit
+
+copy_logs() {
+    if ! command -v virsh > /dev/null 2>&1; then
+        return 0
+    fi
+
+    virsh list --all > ${LOG_DIR}/libvirt_logs/list.txt
+    virsh list --all --name | while read vm; do
+        if [ "$vm" != "" ]; then
+            virsh dumpxml "$vm" > ${LOG_DIR}/libvirt_logs/"$vm".txt
+        fi
+    done
+
+}
+
+copy_logs

roles/tenks-diagnostics/files/get_logs.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# NOTE(wszumski): This has been adapted from tests/get_logs.sh in Kayobe.
+
+# Environment variables:
+# $LOG_DIR is the directory to copy logs to.
+# $CONFIG_DIR is the directory to copy configuration from.
+
+set +o errexit
+
+copy_logs() {
+    if [[ -d ${CONFIG_DIR} ]]; then
+        cp -rnL ${CONFIG_DIR}/* ${LOG_DIR}/config/
+    fi
+    cp -rvnL /var/log/* ${LOG_DIR}/system_logs/
+
+    if [[ -x "$(command -v journalctl)" ]]; then
+        journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.txt
+    fi
+
+    cp -r /etc/sudoers.d ${LOG_DIR}/system_logs/
+    cp /etc/sudoers ${LOG_DIR}/system_logs/sudoers.txt
+
+    df -h > ${LOG_DIR}/system_logs/df.txt
+    free  > ${LOG_DIR}/system_logs/free.txt
+    parted -l > ${LOG_DIR}/system_logs/parted-l.txt
+    mount > ${LOG_DIR}/system_logs/mount.txt
+    env > ${LOG_DIR}/system_logs/env.txt
+    ip address > ${LOG_DIR}/system_logs/ip-address.txt
+    ip route > ${LOG_DIR}/system_logs/ip-route.txt
+
+    iptables-save > ${LOG_DIR}/system_logs/iptables.txt
+
+    if [ `command -v dpkg` ]; then
+        dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt
+    fi
+    if [ `command -v rpm` ]; then
+        rpm -qa > ${LOG_DIR}/system_logs/rpm-qa.txt
+    fi
+
+    # final memory usage and process list
+    ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ${LOG_DIR}/system_logs/ps.txt
+
+    # Rename files to .txt; this is so that when displayed via
+    # logs.openstack.org clicking results in the browser shows the
+    # files, rather than trying to send it to another app or make you
+    # download it, etc.
+    for f in $(find ${LOG_DIR}/{system_logs,libvirt_logs} -name "*.log"); do
+        mv $f ${f/.log/.txt}
+    done
+
+    chmod -R 777 ${LOG_DIR}
+    find ${LOG_DIR}/{system_logs,libvirt_logs} -iname '*.txt' -execdir gzip -f -9 {} \+
+    find ${LOG_DIR}/{system_logs,libvirt_logs} -iname '*.json' -execdir gzip -f -9 {} \+
+}
+
+copy_logs

roles/tenks-diagnostics/tasks/created.yml

@@ -0,0 +1,14 @@
+---
+
+- name: Run diagnostics script
+  script: dump_libvirt_logs.sh
+  register: dump_libvirt_logs_result
+  become: true
+  failed_when: false
+  environment:
+    LOG_DIR: "{{ tenks_diagnostics_log_dir }}"
+    CONFIG_DIR: "{{ tenks_diagnostics_config_dir }}"
+
+- name: Print dump_libvirt_logs.sh output
+  debug:
+    msg: "{{ dump_libvirt_logs_result.stdout }}"

roles/tenks-diagnostics/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include: "{{ tenks_diagnostics_phase }}.yml"

roles/tenks-diagnostics/tasks/post.yml

@@ -0,0 +1,24 @@
+---
+- name: Write host variables to a file
+  copy:
+    content: "{{ hostvars[inventory_hostname] | to_nice_json }}"
+    dest: "{{ tenks_diagnostics_log_dir }}/facts.json"
+
+- name: Run diagnostics script
+  script: get_logs.sh
+  register: get_logs_result
+  become: true
+  failed_when: false
+  environment:
+    LOG_DIR: "{{ tenks_diagnostics_log_dir }}"
+    CONFIG_DIR: "{{ tenks_diagnostics_config_dir }}"
+
+- name: Print get_logs.sh output
+  debug:
+    msg: "{{ get_logs_result.stdout }}"
+
+- name: Download logs to executor
+  synchronize:
+    src: "{{ tenks_diagnostics_log_dir }}/"
+    dest: "{{ tenks_diagnostics_executor_log_dir }}/"
+    mode: pull

roles/tenks-diagnostics/tasks/pre.yml

@@ -0,0 +1,16 @@
+---
+- name: Ensure node log directory exists
+  file:
+    path: "{{ tenks_diagnostics_log_dir }}"
+    state: "directory"
+
+- name: Ensure node log subdirectories exist
+  file:
+    path: "{{ tenks_diagnostics_log_dir }}/{{ item }}"
+    state: "directory"
+    mode: 0777
+  with_items:
+    - "libvirt_logs"
+    - "config"
+    - "system_logs"
+    - "ansible"

zuul.d/base.yaml

@@ -0,0 +1,21 @@
+---
+
+- job:
+    name: tenks-deploy-teardown-base
+    description: |
+      Creates the tenks virtual machines and sets up the virtual BMC daemon.
+      This stops short of enrolling the machines in Ironic. The cluster is
+      then destroyed again.
+    pre-run: playbooks/tenks-deploy-teardown/pre.yml
+    run: playbooks/tenks-deploy-teardown/run.yml
+    post-run: playbooks/tenks-deploy-teardown/post.yml
+    required-projects:
+      - name: openstack/requirements
+    attempts: 1
+    timeout: 1800
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*
+      - ^releasenotes/.*
+      - ^setup.cfg$
+      - ^tox.ini$

zuul.d/jobs.yaml

@@ -7,3 +7,13 @@
     vars:
       tox_envlist: alint
 
+- job:
+    name: tenks-deploy-teardown-centos
+    parent: tenks-deploy-teardown-base
+    nodeset: centos-7
+
+- job:
+    name: tenks-deploy-teardown-ubuntu
+    parent: tenks-deploy-teardown-base
+    nodeset: ubuntu-bionic
+

zuul.d/project.yaml

@@ -9,8 +9,12 @@
       jobs:
         - build-openstack-releasenotes
         - tenks-tox-ansible-lint
+        - tenks-deploy-teardown-centos
+        - tenks-deploy-teardown-ubuntu
     gate:
       queue: tenks
       jobs:
         - build-openstack-releasenotes
         - tenks-tox-ansible-lint
+        - tenks-deploy-teardown-centos
+        - tenks-deploy-teardown-ubuntu