Force the use of the new tempest testing method
The old tempest execution method was deprecated in Newton. We need to shift over to using the new method which is more reliable and easier to configure. Change-Id: Idb750cd75f82b4af7cc7c66ce3dca21d2a7066c8
This commit is contained in:
Jesse Pretorius
parent
9596a072b1
commit
e33c72c36e
@ -22,7 +22,7 @@ tempest_package_state: "latest"
|
||||
tempest_pip_package_state: "latest"
|
||||
|
||||
# Toggle whether tempest actually executes
|
||||
tempest_run: False
|
||||
tempest_run: yes
|
||||
|
||||
tempest_git_repo: https://git.openstack.org/openstack/tempest
|
||||
tempest_git_install_branch: master
|
||||
|
||||
@ -0,0 +1,10 @@
|
||||
---
|
||||
upgrade:
|
||||
- The ``openstack_tempest_gate.sh`` script has been removed
|
||||
as it requires the use of the ``run_tempest.sh`` script
|
||||
which has been deprecated in Tempest. In order to
|
||||
facilitate the switch, the default for the variable
|
||||
``tempest_run`` has been set to ``yes``, forcing the role
|
||||
to execute tempest by default. This default can be changed
|
||||
by overriding the value to ``no``. The test whitelist may
|
||||
be set through the list variable ``tempest_test_whitelist``.
|
||||
@ -13,13 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- include: tempest_install_apt.yml
|
||||
static: no
|
||||
when: ansible_pkg_mgr == 'apt'
|
||||
|
||||
- include: tempest_install_yum.yml
|
||||
static: no
|
||||
when: ansible_pkg_mgr == 'yum'
|
||||
- include: "tempest_install_{{ ansible_pkg_mgr }}.yml"
|
||||
|
||||
- name: Create developer mode constraint file
|
||||
copy:
|
||||
@ -56,26 +50,6 @@
|
||||
state: absent
|
||||
when: tempest_get_venv | changed
|
||||
|
||||
# TODO(odyssey4me):
|
||||
# Once the execution of tempest no longer uses run_tempest.sh
|
||||
# then this task may be removed. This task only executes when
|
||||
# the variable 'tempest_run' is a boolean false, which will be
|
||||
# the case in roles which are using the legacy method.
|
||||
- name: Get tempest from git
|
||||
git:
|
||||
repo: "{{ tempest_git_repo }}"
|
||||
dest: "{{ tempest_venv_bin | dirname }}"
|
||||
version: "{{ tempest_git_install_branch }}"
|
||||
clone: "yes"
|
||||
update: "yes"
|
||||
accept_hostkey: "yes"
|
||||
force: "yes"
|
||||
when: not tempest_run | bool
|
||||
register: git_clone
|
||||
until: git_clone | success
|
||||
retries: 5
|
||||
delay: 2
|
||||
|
||||
- name: Create tempest venv dir
|
||||
file:
|
||||
path: "{{ tempest_venv_bin | dirname }}"
|
||||
|
||||
@ -31,10 +31,6 @@
|
||||
set_fact:
|
||||
tempest_admin_tenant_id: "{{ keystone_facts.id }}"
|
||||
|
||||
# TODO(odyssey4me):
|
||||
# Remove the creation of the etc and lock directories
|
||||
# when all consumers of this role are no longer using
|
||||
# the old execution method.
|
||||
- name: Create tempest directories
|
||||
file:
|
||||
path: "{{ item.path }}"
|
||||
@ -43,11 +39,8 @@
|
||||
group: "root"
|
||||
mode: "{{ item.mode|default('0755') }}"
|
||||
with_items:
|
||||
- path: "{{ tempest_venv_bin | dirname }}/locks"
|
||||
mode: "0777"
|
||||
- path: "{{ tempest_log_dir }}"
|
||||
- path: "{{ ansible_env.HOME }}/.tempest/etc"
|
||||
- path: "{{ tempest_venv_bin | dirname }}/etc"
|
||||
- path: "{{ tempest_image_dir }}"
|
||||
|
||||
- name: Image(s) download
|
||||
@ -58,21 +51,15 @@
|
||||
with_items: "{{ tempest_images }}"
|
||||
when: tempest_service_available_glance | bool
|
||||
|
||||
# TODO(odyssey4me):
|
||||
# Remove the placement to /opt/tempest/etc/ when all
|
||||
# consumers of the role are no longer using the old method.
|
||||
- name: Copy tempest config
|
||||
config_template:
|
||||
src: "tempest.conf.j2"
|
||||
dest: "{{ item }}"
|
||||
dest: "{{ ansible_env.HOME }}/.tempest/etc/tempest.conf"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
mode: "0644"
|
||||
config_overrides: "{{ tempest_tempest_conf_overrides }}"
|
||||
config_type: "ini"
|
||||
with_items:
|
||||
- "{{ tempest_venv_bin | dirname }}/etc/tempest.conf"
|
||||
- "{{ ansible_env.HOME }}/.tempest/etc/tempest.conf"
|
||||
|
||||
- name: Initialise tempest workspace
|
||||
shell: |
|
||||
@ -102,11 +89,3 @@
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
dest: "{{ tempest_test_whitelist_file_path }}"
|
||||
|
||||
- name: Generate tempest Config
|
||||
template:
|
||||
src: "openstack_tempest_gate.sh.j2"
|
||||
dest: "/opt/openstack_tempest_gate.sh"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
mode: "0755"
|
||||
|
||||
@ -1,222 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2014, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Script for running gate tests. Initially very sparse
|
||||
# additional projects and test types will be added over time.
|
||||
|
||||
|
||||
# -------------------- Shell Options -------------------------
|
||||
set -x
|
||||
|
||||
|
||||
# -------------------- Parameters -------------------------
|
||||
# The only parameters this script takes is the names of the test lists
|
||||
# to use:
|
||||
# ./$0 <test_list_name> ...
|
||||
#
|
||||
# If a name is not supplied scenario will be used.
|
||||
# If multiple lists are given, the resulting tests are combined and
|
||||
# duplicates are removed.
|
||||
|
||||
test_lists=${*:-scenario}
|
||||
testr_output_lines=${testr_output_lines:-100}
|
||||
RUN_TEMPEST_OPTS=${RUN_TEMPEST_OPTS:-''}
|
||||
TESTR_OPTS=${TESTR_OPTS:-''}
|
||||
|
||||
|
||||
# -------------------- Functions -------------------------
|
||||
# Test list functions. Each tempest test scenario (eg scenario, api, smoke,
|
||||
# defcore) should have a function here to generate the list of tests that
|
||||
# should be run. Each function takes in the full list of tempest tests and
|
||||
# should output a filtered list.
|
||||
|
||||
function gen_test_list_api {
|
||||
# filter test list to produce list of tests to use.
|
||||
egrep 'tempest\.api\.(identity|image|volume|network|compute|object_storage)'\
|
||||
|grep -vi xml
|
||||
}
|
||||
|
||||
# Run selected scenario tests
|
||||
function gen_test_list_scenario {
|
||||
# network tests have been removed due to
|
||||
# https://bugs.launchpad.net/openstack-ansible/+bug/1425255
|
||||
# TODO: add them back once the bug has been fixed
|
||||
egrep 'tempest\.scenario\.test_(minimum|swift|server|dashboard)_basic(_ops)?'
|
||||
}
|
||||
|
||||
# Run heat-api tests
|
||||
function gen_test_list_heat_api {
|
||||
# basic orchestration api tests
|
||||
egrep 'tempest\.api\.orchestration\.stacks\.test_non_empty_stack'
|
||||
}
|
||||
|
||||
# Run cinder backup api tests
|
||||
function gen_test_list_cinder_backup {
|
||||
egrep 'tempest\.api\.volume\.admin\.test_volumes_backup'
|
||||
}
|
||||
|
||||
# Run the cinder multi-backend tests
|
||||
function gen_test_list_cinder_multi_backend {
|
||||
egrep 'tempest\.api\.volume\.admin\.test_multi_backend'
|
||||
}
|
||||
|
||||
# Run ceilometer api tests
|
||||
function gen_test_list_ceilometer_api {
|
||||
egrep 'tempest\.api\.telemetry'
|
||||
}
|
||||
|
||||
# Run smoke tests
|
||||
function gen_test_list_smoke {
|
||||
# this specific test fails frequently and is making our multi node nightly
|
||||
# job unstable (see bug in gen_test_list_scenario function)
|
||||
# TODO: re-add back in once the specific issue is identified and corrected
|
||||
grep smoke | grep -v tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_update_router_admin_state
|
||||
}
|
||||
|
||||
# Run designate scenario tests
|
||||
function gen_test_list_designate_scenario {
|
||||
egrep 'designate_tempest_plugin\.tests\.scenario'
|
||||
}
|
||||
|
||||
# Run all tests
|
||||
function gen_test_list_all {
|
||||
cat
|
||||
}
|
||||
|
||||
# Generate test list from official defcore/refstack spec
|
||||
function gen_test_list_defcore {
|
||||
branch=${1:-kilo}
|
||||
string=$(python <<END
|
||||
import json
|
||||
import requests
|
||||
|
||||
branch="$branch"
|
||||
map = {
|
||||
'kilo': '2016.01',
|
||||
'liberty': '2016.01'
|
||||
}
|
||||
|
||||
url=("https://raw.githubusercontent.com/openstack/"
|
||||
"defcore/master/%(version)s.json"
|
||||
% {'version': map[branch]}
|
||||
)
|
||||
response = requests.get(url)
|
||||
for capability in response.json()['capabilities'].values():
|
||||
for test in capability['tests']:
|
||||
print test
|
||||
END
|
||||
)
|
||||
|
||||
read -a test_list <<<$string
|
||||
parse_cmd="grep "
|
||||
for item in ${test_list[@]}; do
|
||||
parse_cmd+="-e $item "
|
||||
done
|
||||
|
||||
$parse_cmd
|
||||
}
|
||||
|
||||
# defcore tests for kilo
|
||||
function gen_test_list_defcore_kilo {
|
||||
gen_test_list_defcore kilo
|
||||
}
|
||||
|
||||
# defcore tests for liberty
|
||||
function gen_test_list_defcore_liberty {
|
||||
gen_test_list_defcore liberty
|
||||
}
|
||||
|
||||
function exit_msg {
|
||||
echo $1
|
||||
exit $2
|
||||
}
|
||||
|
||||
|
||||
# -------------------- Main -------------------------
|
||||
|
||||
available_test_lists=$(compgen -A function|sed -n '/^gen_test_list_/s/gen_test_list_//p')
|
||||
|
||||
for test_list_name in $test_lists; do
|
||||
grep $test_list_name <<<$available_test_lists ||\
|
||||
exit_msg "$test_list_name is not a valid test list, available test lists: $available_test_lists" 1
|
||||
done
|
||||
|
||||
# work in tempest directory
|
||||
pushd {{ tempest_venv_bin | dirname }}
|
||||
|
||||
# Load the tempest venv for a tempest run
|
||||
source {{ tempest_venv_bin | dirname }}/bin/activate
|
||||
|
||||
# read creds into environment
|
||||
source /root/openrc
|
||||
|
||||
# create testr repo - required for testr to do anything
|
||||
testr init &>/dev/null ||:
|
||||
|
||||
# Get list of available tests.
|
||||
# lines 1-$testr_output_lines are output to stdout, all lines are written to
|
||||
# full_test_list.
|
||||
set -o pipefail
|
||||
testr list-tests |tee >(sed -n 1,${testr_output_lines}p) >full_test_list ||\
|
||||
exit_msg "Failed to generate test list" $?
|
||||
set +o pipefail
|
||||
|
||||
# Check the full test list is not empty
|
||||
[[ -s full_test_list ]] || exit_msg "No tests found" 1
|
||||
|
||||
# Write filter test list using selected function. The full test list is
|
||||
# pre-filtered to only include test lines, this saves adding that filter to
|
||||
# every test list function.
|
||||
truncate --size 0 tmp_test_list
|
||||
for test_list_name in ${test_lists}; do
|
||||
egrep '^(tempest|.*_tempest_plugin)\.' < full_test_list \
|
||||
| gen_test_list_${test_list_name} >> tmp_test_list \
|
||||
|| exit_msg "Filter $test_list_name failed. Output: $(cat test_list)" 1
|
||||
done
|
||||
|
||||
# Eliminate duplicate tests
|
||||
awk '!seen[$0]++' tmp_test_list > test_list
|
||||
rm tmp_test_list
|
||||
|
||||
# Check the filtered test list is not empty
|
||||
[[ -s test_list ]] || exit_msg "No tests remain after filtering" 1
|
||||
|
||||
test_list_summary="${test_lists} ($(wc -l <test_list) tests)"
|
||||
|
||||
echo "Using test list $test_list_summary"
|
||||
|
||||
# execute chosen tests with pretty output
|
||||
./run_tempest.sh --no-virtual-env ${RUN_TEMPEST_OPTS} -- --load-list test_list ${TESTR_OPTS};
|
||||
result=$?
|
||||
|
||||
# Create junit xml report from test results
|
||||
testr last --subunit | subunit2junitxml > tempest_results.xml
|
||||
cp tempest_results.xml /tmp
|
||||
if [ -d "{{ tempest_log_dir }}" ];then
|
||||
# Copy the log to the tempest_log_dir logging directory
|
||||
cp tempest_results.xml {{ tempest_log_dir }}/tempest_results_$(date +%Y%d%m-%H%M%S).xml
|
||||
fi
|
||||
popd
|
||||
|
||||
if [[ $result == 0 ]]; then
|
||||
echo "TEMPEST PASS $test_list_summary"
|
||||
else
|
||||
echo "TEMPEST FAIL $test_list_summary"
|
||||
fi
|
||||
|
||||
# Deactivate the venv after run
|
||||
deactivate || true
|
||||
|
||||
exit $result
|
||||
Loading…
Reference in New Issue
Block a user