Merge "Rework functional test runner"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
8e9c54d01a
@ -372,6 +372,14 @@ In order to create a test case, follow these steps:
|
||||
* To exit early and indicate a success, touch a file
|
||||
``/tmp/dib-test-should-fail`` in the image chroot, then exit 1.
|
||||
|
||||
Tests are run with ``tools/run_functests.sh``. Running
|
||||
``run_functests.sh -l`` will show available tests (the example above
|
||||
would be called ``apt-sources/test-case-1``, for example). Specify
|
||||
your test (or a series of tests as separate arguments) on the command
|
||||
line to run it. If it should not be run as part of the default CI
|
||||
run, you can submit a change with it added to ``DEFAULT_SKIP_TESTS``
|
||||
in that file.
|
||||
|
||||
python
|
||||
""""""
|
||||
|
||||
|
||||
12
tests/README.rst
Normal file
12
tests/README.rst
Normal file
@ -0,0 +1,12 @@
|
||||
DIB Testing
|
||||
===========
|
||||
|
||||
There are several interfaces for testing various parts of DIB
|
||||
|
||||
* run_dib_library_tests.sh : run unit-tests for internal
|
||||
diskimage-builder library functions
|
||||
|
||||
* run_functests.sh : run functional testing provided by elements
|
||||
|
||||
* run_output_format_test.sh : run a test of image conversion
|
||||
(requires docker)
|
||||
@ -1,23 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
set -o pipefail
|
||||
|
||||
source $(dirname $0)/test_functions.bash
|
||||
|
||||
test_formats="tar raw qcow2 docker aci"
|
||||
for binary in qemu-img docker ; do
|
||||
if [ -z "$(which $binary)" ]; then
|
||||
echo "Warning: No $binary binary found, cowardly refusing to run tests."
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
for format in '' $test_formats; do
|
||||
build_test_image $format
|
||||
echo "Test passed for output formats '$format'."
|
||||
done
|
||||
|
||||
combined_format=$(echo $test_formats | tr ' ' ',')
|
||||
build_test_image $combined_format
|
||||
echo "Test passed for output format '$combined_format'."
|
||||
@ -1,13 +1,179 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
element=${1:-}
|
||||
BASE_DIR=$(cd $(dirname "$0")/.. && pwd)
|
||||
export DIB_ELEMENTS=$BASE_DIR/elements
|
||||
export DIB_CMD=$BASE_DIR/bin/disk-image-create
|
||||
|
||||
if [ -z $element ]; then
|
||||
$(dirname $0)/image_output_formats.bash
|
||||
#
|
||||
# Default skip tests
|
||||
#
|
||||
# For time reasons, we do not run these tests by default; i.e. these
|
||||
# tests are not run by "tox -e func" in the gate.
|
||||
#
|
||||
DEFAULT_SKIP_TESTS=(
|
||||
# we run version pinned test in gate (this just runs latest)
|
||||
fedora/build-succeeds
|
||||
)
|
||||
|
||||
# run_disk_element_test <test_element> <element>
|
||||
# Run a disk-image-build .tar build of ELEMENT including any elements
|
||||
# specified by TEST_ELEMENT
|
||||
function run_disk_element_test() {
|
||||
local test_element=$1
|
||||
local element=$2
|
||||
local dest_dir=$(mktemp -d)
|
||||
|
||||
trap "rm -rf $dest_dir /tmp/dib-test-should-fail" EXIT
|
||||
|
||||
if break="after-error" break_outside_target=1 \
|
||||
break_cmd="cp \$TMP_MOUNT_PATH/tmp/dib-test-should-fail /tmp/ 2>&1 > /dev/null || true" \
|
||||
ELEMENTS_PATH=$DIB_ELEMENTS:$DIB_ELEMENTS/$element/test-elements \
|
||||
$DIB_CMD -t tar -o $dest_dir/image -n $element $test_element; then
|
||||
if ! [ -f "$dest_dir/image.tar" ]; then
|
||||
echo "Error: Build failed for element: $element, test-element: $test_element."
|
||||
echo "No image $dest_dir/image.tar found!"
|
||||
exit 1
|
||||
else
|
||||
if tar -tf $dest_dir/image.tar | grep -q /tmp/dib-test-should-fail; then
|
||||
echo "Error: Element: $element, test-element $test_element should have failed, but passed."
|
||||
exit 1
|
||||
else
|
||||
echo "PASS: Element $element, test-element: $test_element"
|
||||
fi
|
||||
$(dirname $0)/test_elements.bash $element
|
||||
fi
|
||||
else
|
||||
if [ -f "/tmp/dib-test-should-fail" ]; then
|
||||
echo "PASS: Element $element, test-element: $test_element"
|
||||
else
|
||||
echo "Error: Build failed for element: $element, test-element: $test_element."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
trap EXIT
|
||||
rm -rf $dest_dir /tmp/dib-test-should-fail
|
||||
}
|
||||
|
||||
# run_ramdisk_element_test <test_element> <element>
|
||||
# Run a disk-image-builder default build of ELEMENT including any
|
||||
# elements specified by TEST_ELEMENT
|
||||
function run_ramdisk_element_test() {
|
||||
local test_element=$1
|
||||
local element=$2
|
||||
local dest_dir=$(mktemp -d)
|
||||
|
||||
if ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \
|
||||
$DIB_CMD -o $dest_dir/image $element $test_element; then
|
||||
# TODO(dtantsur): test also kernel presence once we sort out its naming
|
||||
# problem (vmlinuz vs kernel)
|
||||
if ! [ -f "$dest_dir/image.initramfs" ]; then
|
||||
echo "Error: Build failed for element: $element, test-element: $test_element."
|
||||
echo "No image $dest_dir/image.initramfs found!"
|
||||
exit 1
|
||||
else
|
||||
echo "PASS: Element $element, test-element: $test_element"
|
||||
fi
|
||||
else
|
||||
echo "Error: Build failed for element: $element, test-element: $test_element."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
#
|
||||
# run_functests.sh
|
||||
# run the functional tests for dib elements
|
||||
#
|
||||
|
||||
# find elements that have functional test elements. TESTS will be an
|
||||
# array with each value being "element/test-element"
|
||||
TESTS=()
|
||||
for e in $DIB_ELEMENTS/*/test-elements/*; do
|
||||
test_element=$(echo $e | awk 'BEGIN {FS="/"}{print $NF}')
|
||||
element=$(echo $e | awk 'BEGIN {FS="/"}{print $(NF-2)}')
|
||||
TESTS+=("$element/$test_element")
|
||||
done
|
||||
|
||||
while getopts ":hl" opt; do
|
||||
case $opt in
|
||||
h)
|
||||
echo "run_functests.sh [-h] [-l] <test> <test> ..."
|
||||
echo " -h : show this help"
|
||||
echo " -l : list available tests"
|
||||
echo " <test> : functional test to run"
|
||||
echo " Special test 'all' will run all tests"
|
||||
exit 0
|
||||
;;
|
||||
l)
|
||||
echo "The available functional tests are:"
|
||||
echo
|
||||
for t in ${TESTS[@]}; do
|
||||
echo " $t"
|
||||
done
|
||||
echo
|
||||
exit 0
|
||||
;;
|
||||
\?)
|
||||
echo "Invalid option: -$OPTARG"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND-1))
|
||||
|
||||
# cull the list of tests to run into TESTS_TO_RUN
|
||||
TESTS_TO_RUN=()
|
||||
title=""
|
||||
if [[ -z "$@" ]]; then
|
||||
# remove the skipped tests
|
||||
title="Running default tests:"
|
||||
for test in "${TESTS[@]}"; do
|
||||
if [[ " ${DEFAULT_SKIP_TESTS[@]} " =~ " ${test} " ]]; then
|
||||
continue
|
||||
else
|
||||
TESTS_TO_RUN+=("${test}")
|
||||
fi
|
||||
done
|
||||
elif [[ $1 == "all" ]]; then
|
||||
title="Running all tests:"
|
||||
TESTS_TO_RUN=("${TESTS[@]}")
|
||||
else
|
||||
title="Running specified tests:"
|
||||
for test in $@; do
|
||||
if [[ ! " ${TESTS[@]} " =~ " ${test} " ]]; then
|
||||
echo "${test} : not a known test (see -l)"
|
||||
exit 1
|
||||
fi
|
||||
TESTS_TO_RUN+=("${test}")
|
||||
done
|
||||
fi
|
||||
|
||||
# print a little status info
|
||||
echo "------"
|
||||
echo ${title}
|
||||
for test in "${TESTS_TO_RUN[@]}"; do
|
||||
echo " ${test}"
|
||||
done
|
||||
echo "------"
|
||||
|
||||
for test in "${TESTS_TO_RUN[@]}"; do
|
||||
# from above; each array value is element/test_element. split it
|
||||
# back up
|
||||
element=${test%/*}
|
||||
test_element=${test#*/}
|
||||
|
||||
# tests default to disk-based, but "element-type" can optionally
|
||||
# override that
|
||||
element_type=disk
|
||||
element_type_override=$DIB_ELEMENTS/${element}/test-elements/${test_element}/element-type
|
||||
if [ -f ${element_type_override} ]; then
|
||||
element_type=$(cat ${element_type_override})
|
||||
fi
|
||||
|
||||
echo "Running $test ($element_type)"
|
||||
run_${element_type}_element_test $test_element $element
|
||||
done
|
||||
|
||||
echo "Tests passed!"
|
||||
|
||||
77
tests/run_output_format_test.sh
Executable file
77
tests/run_output_format_test.sh
Executable file
@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
#
|
||||
# run_output_format_test.sh
|
||||
#
|
||||
# Use docker to test generation of various output formats.
|
||||
#
|
||||
|
||||
BASE_DIR=$(cd $(dirname "$0")/.. && pwd)
|
||||
export DIB_ELEMENTS=$BASE_DIR/elements
|
||||
export TEST_ELEMENTS=$BASE_DIR/tests/elements
|
||||
export DIB_CMD=$BASE_DIR/bin/disk-image-create
|
||||
|
||||
function build_test_image() {
|
||||
format=${1:-}
|
||||
|
||||
if [ -n "$format" ]; then
|
||||
type_arg="-t $format"
|
||||
else
|
||||
type_arg=
|
||||
format="qcow2"
|
||||
fi
|
||||
dest_dir=$(mktemp -d)
|
||||
base_dest=$(basename $dest_dir)
|
||||
|
||||
trap "rm -rf $dest_dir; docker rmi $base_dest/image" EXIT
|
||||
|
||||
ELEMENTS_PATH=$DIB_ELEMENTS:$TEST_ELEMENTS \
|
||||
$DIB_CMD -x $type_arg --docker-target=$base_dest/image \
|
||||
-o $dest_dir/image -n fake-os
|
||||
|
||||
format=$(echo $format | tr ',' ' ')
|
||||
for format in $format; do
|
||||
if [ $format != 'docker' ]; then
|
||||
img_path="$dest_dir/image.$format"
|
||||
if ! [ -f "$img_path" ]; then
|
||||
echo "Error: No image with name $img_path found!"
|
||||
exit 1
|
||||
else
|
||||
echo "Found image $img_path."
|
||||
fi
|
||||
else
|
||||
if ! docker images | grep $base_dest/image ; then
|
||||
echo "Error: No docker image with name $base_dest/image found!"
|
||||
exit 1
|
||||
else
|
||||
echo "Found docker image $base_dest/image"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
trap EXIT
|
||||
rm -rf $dest_dir
|
||||
if docker images | grep $base_dest/image ; then
|
||||
docker rmi $base_dest/image
|
||||
fi
|
||||
}
|
||||
|
||||
test_formats="tar raw qcow2 docker aci"
|
||||
for binary in qemu-img docker ; do
|
||||
if [ -z "$(which $binary)" ]; then
|
||||
echo "Warning: No $binary binary found, cowardly refusing to run tests."
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
for format in '' $test_formats; do
|
||||
build_test_image $format
|
||||
echo "Test passed for output formats '$format'."
|
||||
done
|
||||
|
||||
combined_format=$(echo $test_formats | tr ' ' ',')
|
||||
build_test_image $combined_format
|
||||
echo "Test passed for output format '$combined_format'."
|
||||
@ -1,33 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
set -o pipefail
|
||||
|
||||
basedir=$(dirname $0)
|
||||
requested_element=${1:-}
|
||||
source $basedir/test_functions.bash
|
||||
|
||||
function run_on_element {
|
||||
test_element=$1
|
||||
# our element name is two dirs up
|
||||
local element_name=$(basename $(dirname $(dirname $test_element)))
|
||||
local element_type=disk
|
||||
if [ -f "$test_element/element-type" ]; then
|
||||
element_type=$(cat "$test_element/element-type")
|
||||
fi
|
||||
run_${element_type}_element_test "$(basename $test_element)" "$element_name"
|
||||
}
|
||||
|
||||
if [ -z $requested_element ]; then
|
||||
for test_element in $basedir/../elements/*/test-elements/*; do
|
||||
if [ -d "$test_element" ]; then
|
||||
run_on_element "$test_element"
|
||||
fi
|
||||
done
|
||||
else
|
||||
for test_element in $basedir/../elements/$requested_element/test-elements/*; do
|
||||
if [ -d "$test_element" ]; then
|
||||
run_on_element "$test_element"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
@ -1,108 +0,0 @@
|
||||
export TEST_ELEMENTS=$(dirname $0)/elements
|
||||
export DIB_ELEMENTS=$(dirname $0)/../elements
|
||||
export DIB_CMD=$(dirname $0)/../bin/disk-image-create
|
||||
|
||||
function build_test_image() {
|
||||
format=${1:-}
|
||||
|
||||
if [ -n "$format" ]; then
|
||||
type_arg="-t $format"
|
||||
else
|
||||
type_arg=
|
||||
format="qcow2"
|
||||
fi
|
||||
dest_dir=$(mktemp -d)
|
||||
base_dest=$(basename $dest_dir)
|
||||
|
||||
trap "rm -rf $dest_dir; docker rmi $base_dest/image" EXIT
|
||||
|
||||
ELEMENTS_PATH=$DIB_ELEMENTS:$TEST_ELEMENTS \
|
||||
$DIB_CMD -x $type_arg --docker-target=$base_dest/image \
|
||||
-o $dest_dir/image -n fake-os
|
||||
|
||||
format=$(echo $format | tr ',' ' ')
|
||||
for format in $format; do
|
||||
if [ $format != 'docker' ]; then
|
||||
img_path="$dest_dir/image.$format"
|
||||
if ! [ -f "$img_path" ]; then
|
||||
echo "Error: No image with name $img_path found!"
|
||||
exit 1
|
||||
else
|
||||
echo "Found image $img_path."
|
||||
fi
|
||||
else
|
||||
if ! docker images | grep $base_dest/image ; then
|
||||
echo "Error: No docker image with name $base_dest/image found!"
|
||||
exit 1
|
||||
else
|
||||
echo "Found docker image $base_dest/image"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
trap EXIT
|
||||
rm -rf $dest_dir
|
||||
if docker images | grep $base_dest/image ; then
|
||||
docker rmi $base_dest/image
|
||||
fi
|
||||
}
|
||||
|
||||
function run_disk_element_test() {
|
||||
test_element=$1
|
||||
element=$2
|
||||
|
||||
dest_dir=$(mktemp -d)
|
||||
|
||||
trap "rm -rf $dest_dir /tmp/dib-test-should-fail" EXIT
|
||||
|
||||
if break="after-error" break_outside_target=1 \
|
||||
break_cmd="cp \$TMP_MOUNT_PATH/tmp/dib-test-should-fail /tmp/ 2>&1 > /dev/null || true" \
|
||||
ELEMENTS_PATH=$DIB_ELEMENTS:$DIB_ELEMENTS/$element/test-elements \
|
||||
$DIB_CMD -t tar -o $dest_dir/image -n $element $test_element; then
|
||||
if ! [ -f "$dest_dir/image.tar" ]; then
|
||||
echo "Error: Build failed for element: $element, test-element: $test_element."
|
||||
echo "No image $dest_dir/image.tar found!"
|
||||
exit 1
|
||||
else
|
||||
if tar -tf $dest_dir/image.tar | grep -q /tmp/dib-test-should-fail; then
|
||||
echo "Error: Element: $element, test-element $test_element should have failed, but passed."
|
||||
exit 1
|
||||
else
|
||||
echo "PASS: Element $element, test-element: $test_element"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [ -f "/tmp/dib-test-should-fail" ]; then
|
||||
echo "PASS: Element $element, test-element: $test_element"
|
||||
else
|
||||
echo "Error: Build failed for element: $element, test-element: $test_element."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
trap EXIT
|
||||
rm -rf $dest_dir /tmp/dib-test-should-fail
|
||||
}
|
||||
|
||||
function run_ramdisk_element_test() {
|
||||
test_element=$1
|
||||
element=$2
|
||||
|
||||
dest_dir=$(mktemp -d)
|
||||
|
||||
if ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \
|
||||
$DIB_CMD -o $dest_dir/image $element $test_element; then
|
||||
# TODO(dtantsur): test also kernel presence once we sort out its naming
|
||||
# problem (vmlinuz vs kernel)
|
||||
if ! [ -f "$dest_dir/image.initramfs" ]; then
|
||||
echo "Error: Build failed for element: $element, test-element: $test_element."
|
||||
echo "No image $dest_dir/image.initramfs found!"
|
||||
exit 1
|
||||
else
|
||||
echo "PASS: Element $element, test-element: $test_element"
|
||||
fi
|
||||
else
|
||||
echo "Error: Build failed for element: $element, test-element: $test_element."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user