6c5e07e31c
Recently, we added new job "gate-rally-dsvm-verify" for Rally[1]. This job implied functional testing integration Rally with Tempest[2]. This patch adds tests/ci/rally-verify.sh script, which is used by new job. rally-verify.sh does: - tempest installation - run "rally verify start" twice and print results - compare results of two verifications - list verifications - generate html page based on results To implement gate-rally-dsvm-verify some changes were requered to existing Rally code: - Added ability for rally/ui/utils.py to accept arguments to render html-pages - Fixed logging debug-messages in tempest verifier - Fixed check "is debug mode turned on or not"(also, added hacking rule for it) TODO for future patches: - add launch of rally task for Tempest - add launch of random test set - add check for successful tests [1] https://review.openstack.org/#/c/137232 [2] https://www.mirantis.com/blog/rally-openstack-tempest-testing-made-simpler Closes-Bug: #1400465 Closes-Bug: #1400518 Change-Id: I8e1fbab22c2da109bbc442f040fe259e5d22a62a
97 lines
3.0 KiB
Bash
Executable File
97 lines
3.0 KiB
Bash
Executable File
#!/bin/bash -ex
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compli$OUT with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# This script is executed by post_test_hook function in desvstack gate.
|
|
|
|
RESULTS_DIR="rally-verify"
|
|
|
|
env
|
|
set -o pipefail
|
|
set +e
|
|
|
|
mkdir -p ${RESULTS_DIR}/extra
|
|
|
|
|
|
# Check deployment
|
|
rally use deployment --deployment devstack
|
|
rally deployment check
|
|
|
|
function do_status {
|
|
if [[ ${1} != 0 ]]
|
|
then
|
|
echo "fail"
|
|
else
|
|
echo "pass"
|
|
fi
|
|
}
|
|
|
|
declare -a RESULTS
|
|
|
|
rally-manage --rally-debug tempest install > ${RESULTS_DIR}/tempest_installation.txt 2>&1
|
|
RESULTS+="install=$(do_status $?) "
|
|
|
|
gzip -9 ${RESULTS_DIR}/tempest_installation.txt
|
|
|
|
# Run to verification for one SET_NAME and then compare them.
|
|
SET_NAME="compute"
|
|
|
|
function do_verification {
|
|
OUTPUT_FILE=${RESULTS_DIR}/${1}_verification_${SET_NAME}_set.txt
|
|
rally --rally-debug verify start --set ${SET_NAME} > ${OUTPUT_FILE} 2>&1
|
|
RESULTS+="v${1}=$(do_status $?) "
|
|
gzip -9 ${OUTPUT_FILE}
|
|
source ~/.rally/globals && VERIFICATIONS[${1}]=${RALLY_VERIFICATION}
|
|
|
|
# Check different "rally verify" commands, which displays verification results
|
|
for OUTPUT_FORMAT in "html" "json"
|
|
do
|
|
OUTPUT_FILE=${RESULTS_DIR}/${1}_verify_results.${OUTPUT_FORMAT}
|
|
rally verify results --uuid ${RALLY_VERIFICATION} --${OUTPUT_FORMAT} --output-file ${OUTPUT_FILE}
|
|
RESULTS+="vr_${1}_${OUTPUT_FORMAT}=$(do_status $?) "
|
|
gzip -9 ${OUTPUT_FILE}
|
|
done
|
|
|
|
rally verify show --uuid ${RALLY_VERIFICATION} > ${RESULTS_DIR}/${1}_verify_show.txt
|
|
RESULTS+="vs_${1}=$(do_status $?) "
|
|
gzip -9 ${RESULTS_DIR}/${1}_verify_show.txt
|
|
|
|
rally verify show --uuid ${RALLY_VERIFICATION} --detailed > ${RESULTS_DIR}/${1}_verify_show_detailed.txt
|
|
RESULTS+="vsd_${1}=$(do_status $?) "
|
|
gzip -9 ${RESULTS_DIR}/${1}_verify_show_detailed.txt
|
|
}
|
|
|
|
do_verification 1
|
|
do_verification 2
|
|
|
|
rally verify list > ${RESULTS_DIR}/verify_list.txt
|
|
RESULTS+="l=$(do_status $?) "
|
|
gzip -9 ${RESULTS_DIR}/verify_list.txt
|
|
|
|
# Compare and save results in different formats
|
|
for OUTPUT_FORMAT in "csv" "html" "json"
|
|
do
|
|
OUTPUT_FILE=${RESULTS_DIR}/compare_results.${OUTPUT_FORMAT}
|
|
rally --rally-debug verify compare --uuid-1 ${VERIFICATIONS[1]} --uuid-2 ${VERIFICATIONS[2]} --${OUTPUT_FORMAT} --output-file ${OUTPUT_FILE}
|
|
RESULTS+="c_${OUTPUT_FORMAT}=$(do_status $?) "
|
|
gzip -9 ${OUTPUT_FILE}
|
|
done
|
|
|
|
python $BASE/new/rally/rally/ui/utils.py render\
|
|
tests/ci/rally-gate/index_verify.mako ${RESULTS[*]}> ${RESULTS_DIR}/extra/index.html
|
|
|
|
if [[ ${RESULTS[*]} == *"fail"* ]]
|
|
then
|
|
return 1
|
|
fi
|