Execute tempest using updated method

The old method of executing tempest via run_tempest.sh from
the root of the git clone has been deprecated and will soon
be removed.

This patch does the following:

- Moves the tempest execution into the role.
- Provides the ability to set the tests to be executed
  as a list. The list becomes a whitelist which is given
  to tempest to run. The whitelist can be a specific list
  of tests, or line seperated regex.
- Uses the modern execution method, but leaves the old
  method in place to allow other roles to transition.
- Exports the subunit results as raw, xml and html results.

Implements: blueprint testing-direct-tempest
Change-Id: I40f816ead338f90f9537a8494822ff10b06021cb

.gitignore

@@ -61,10 +61,8 @@ ChangeLog
 releasenotes/build
 
 # Test temp files
-tests/plugins
 tests/common
-tests/test.retry
-tests/playbooks
+tests/*.retry
 
 # Vagrant artifacts
 .vagrant

defaults/main.yml

@@ -20,6 +20,9 @@ debug: False
 # # Options are 'present' and 'latest'
 tempest_package_state: "latest"
 
+# Toggle whether tempest actually executes
+tempest_run: False
+
 tempest_git_repo: https://git.openstack.org/openstack/tempest
 tempest_git_install_branch: master
 tempest_requirements_git_repo: https://git.openstack.org/openstack/requirements
@@ -33,7 +36,7 @@ tempest_venv_tag: untagged
 tempest_venv_bin: "/openstack/venvs/tempest-{{ tempest_venv_tag }}/bin"
 tempest_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/tempest.tgz
 
-# The location where the test results will be placed
+# The location where the tempest logs will be placed
 tempest_log_dir: "/var/log/tempest"
 
 ## Tempest Plugins
@@ -45,6 +48,20 @@ tempest_log_dir: "/var/log/tempest"
 #     branch: master
 tempest_plugins: []
 
+# The location where the test whitelist will be placed
+tempest_test_whitelist_file_path: "{{ tempest_venv_bin | dirname }}/workspace/etc/tempest_whitelist.txt"
+
+# Tests to execute:
+# This sets up a list of tests to execute based on what's deployed in the environment.
+# The list gets added to the whitelist which tempest executes.
+tempest_test_whitelist:
+  - "{{ (tempest_service_available_ceilometer | bool) | ternary('tempest.api.telemetry', '') }}"
+  - "{{ (tempest_service_available_heat | bool) | ternary('tempest.api.orchestration.stacks.test_non_empty_stack', '') }}"
+  - "{{ (tempest_service_available_nova | bool) | ternary('tempest.scenario.test_minimum_basic', '') }}"
+  - "{{ (tempest_service_available_nova | bool) | ternary('tempest.scenario.test_server_basic_ops', '') }}"
+  - "{{ (tempest_service_available_swift | bool) | ternary('tempest.scenario.test_object_storage_basic_ops', '') }}"
+  - "{{ (tempest_volume_backup_enabled | bool) | ternary('tempest.api.volume.admin.test_volumes_backup', '') }}"
+
 # Toggle fatal deprecations
 tempest_fatal_deprecations: False
 

tasks/main.yml

@@ -39,3 +39,6 @@
 
 - include: tempest_post_install.yml
   tags: tempest-config
+
+- include: tempest_run.yml
+  when: tempest_run | bool

tasks/tempest_install.yml

@@ -98,7 +98,9 @@
 
 # TODO(odyssey4me):
 # Once the execution of tempest no longer uses run_tempest.sh
-# then this task may be removed.
+# then this task may be removed. This task only executes when
+# the variable 'tempest_run' is a boolean false, which will be
+# the case in roles which are using the legacy method.
 - name: Get tempest from git
   git:
     repo: "{{ tempest_git_repo }}"
@@ -108,6 +110,7 @@
     update: "yes"
     accept_hostkey: "yes"
     force: "yes"
+  when: not tempest_run | bool
   register: git_clone
   until: git_clone | success
   retries: 5

tasks/tempest_post_install.yml

@@ -31,7 +31,11 @@
   set_fact:
     tempest_admin_tenant_id: "{{ keystone_facts.id }}"
 
-- name: Create tempest dir
+# TODO(odyssey4me):
+# Remove the creation of the etc and lock directories
+# when all consumers of this role are no longer using
+# the old execution method.
+- name: Create tempest directories
   file:
     path: "{{ item.path }}"
     state: directory
@@ -39,9 +43,12 @@
     group: "root"
     mode: "{{ item.mode|default('0755') }}"
   with_items:
-    - { path: "{{ tempest_venv_bin | dirname }}/locks", mode: "0777" }
-    - { path: "{{ tempest_venv_bin | dirname }}/etc" }
-    - { path: "{{ tempest_image_dir }}" }
+    - path: "{{ tempest_venv_bin | dirname }}/locks"
+      mode: "0777"
+    - path: "{{ tempest_log_dir }}"
+    - path: "{{ ansible_env.HOME }}/.tempest/etc"
+    - path: "{{ tempest_venv_bin | dirname }}/etc"
+    - path: "{{ tempest_image_dir }}"
 
 - name: Image(s) download
   get_url:
@@ -51,15 +58,39 @@
   with_items: "{{ tempest_images }}"
   when: tempest_service_available_glance | bool
 
+# TODO(odyssey4me):
+# Remove the placement to /opt/tempest/etc/ when all
+# consumers of the role are no longer using the old method.
 - name: Copy tempest config
   config_template:
     src: "tempest.conf.j2"
-    dest: "{{ tempest_venv_bin | dirname }}/etc/tempest.conf"
+    dest: "{{ item }}"
     owner: "root"
     group: "root"
     mode: "0644"
     config_overrides: "{{ tempest_tempest_conf_overrides }}"
     config_type: "ini"
+  with_items:
+    - "{{ tempest_venv_bin | dirname }}/etc/tempest.conf"
+    - "{{ ansible_env.HOME }}/.tempest/etc/tempest.conf"
+
+- name: Initialise tempest workspace
+  shell: |
+    . {{ tempest_venv_bin }}/activate
+    tempest init workspace
+  args:
+    chdir: "{{ tempest_venv_bin | dirname }}"
+    executable: /bin/bash
+
+- name: Generate tempest test whitelist
+  copy:
+    content: |
+      {% for item in tempest_test_whitelist | unique | sort %}
+      {%   if item != '' %}
+      {{ item }}
+      {%   endif %}
+      {% endfor %}
+    dest: "{{ tempest_test_whitelist_file_path }}"
 
 - name: Generate tempest Config
   template:

tasks/tempest_run.yml

@@ -0,0 +1,46 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Execute tempest tests
+  shell: |
+    . {{ tempest_venv_bin }}/activate
+    tempest run --serial --whitelist-file {{ tempest_test_whitelist_file_path }}
+  args:
+    chdir: "{{ tempest_venv_bin | dirname }}/workspace"
+    executable: /bin/bash
+
+- name: Generate raw subunit results
+  shell: |
+    . {{ tempest_venv_bin }}/activate
+    testr last --subunit > tempest_results.subunit
+  args:
+    chdir: "{{ tempest_venv_bin | dirname }}/workspace"
+    executable: /bin/bash
+
+- name: Generate xml subunit results
+  shell: |
+    . {{ tempest_venv_bin }}/activate
+    subunit2junitxml tempest_results.subunit > tempest_results.xml
+  args:
+    chdir: "{{ tempest_venv_bin | dirname }}/workspace"
+    executable: /bin/bash
+
+- name: Generate html subunit results
+  shell: |
+    . {{ tempest_venv_bin }}/activate
+    subunit2html tempest_results.subunit tempest_results.html
+  args:
+    chdir: "{{ tempest_venv_bin | dirname }}/workspace"
+    executable: /bin/bash

tests/os_tempest-overrides.yml

@@ -13,6 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Ensure that tempest executes tests
+tempest_run: yes
+
+# A sample plugin to activate the code path
 tempest_plugins:
   - name: designate-tempest-plugin
     repo: https://git.openstack.org/openstack/designate-tempest-plugin

tests/test-nova-functional.yml

@@ -1,28 +0,0 @@
----
-# Copyright 2015, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Playbook for functional testing of nova
-  hosts: utility_all[0]
-  user: root
-  gather_facts: false
-  tasks:
-    - name: Run tempest
-      shell: |
-        . {{ tempest_venv_bin }}/activate
-        {{ tempest_venv_bin | dirname }}/run_tempest.sh --no-virtual-env ${RUN_TEMPEST_OPTS} tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops
-      environment:
-        RUN_TEMPEST_OPTS: "--serial"
-  vars_files:
-    - common/test-vars.yml

tests/test-tempest-functional.yml

@@ -18,14 +18,29 @@
   user: root
   gather_facts: false
   tasks:
+
     - name: Ensure tempest plugins have been installed
-      command: "{{ tempest_venv_bin }}/pip show {{ item.name }}"
-      with_items: "{{ tempest_plugins }}"
-    - name: Run tempest
       shell: |
-        . {{ tempest_venv_bin }}/activate
-        {{ tempest_venv_bin | dirname }}/run_tempest.sh --no-virtual-env ${RUN_TEMPEST_OPTS} tempest.api.identity.v3
-      environment:
-        RUN_TEMPEST_OPTS: "--serial"
+        if {{ tempest_venv_bin }}/pip show {{ item.name }} > /dev/null; then
+         echo "The plugin {{ item.name }} has been installed."
+        else
+          echo "The plugin {{ item.name }} has NOT been installed."
+          exit 1
+        fi
+      with_items: "{{ tempest_plugins }}"
+
+    - name: List the contents of the tempest_result_path folder
+      command: "ls -1 {{ tempest_venv_bin | dirname }}/workspace/"
+      register: _results_list
+
+    - name: Check whether the expected subunit result files are present
+      assert:
+        that: "item in _results_list.stdout_lines"
+        msg: "{{ item }} must be present in {{ tempest_venv_bin | dirname }}/workspace/"
+      with_items:
+        - "tempest_results.html"
+        - "tempest_results.subunit"
+        - "tempest_results.xml"
+
   vars_files:
     - common/test-vars.yml

tests/test.yml

@@ -37,11 +37,8 @@
 # Install Nova
 - include: common/test-install-nova.yml
 
-# Install Tempest
+# Install and execute Tempest
 - include: common/test-install-tempest.yml
 
-# Test Tempest
+# Check whether Tempest executed successfully
 - include: test-tempest-functional.yml
-
-# Test Nova
-- include: test-nova-functional.yml

vars/redhat-7.yml

@@ -15,3 +15,4 @@
 
 tempest_distro_packages:
   - git
+  - rsync

vars/ubuntu-14.04.yml

@@ -15,3 +15,4 @@
 
 tempest_distro_packages:
   - git-core
+  - rsync

vars/ubuntu-16.04.yml

@@ -15,3 +15,4 @@
 
 tempest_distro_packages:
   - git-core
+  - rsync