airship
/
divingbell
Code Issues Proposed changes
divingbell/tools/gate/scripts/020-test-divingbell.sh

1802 lines
56 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# TODO: Convert to use new/common gate scripts when available
set -e
NAME=divingbell
: ${LOGS_DIR:=/var/log}
: ${LOGS_SUBDIR:=${LOGS_DIR}/${NAME}/$(date +"%m-%d-%y_%H:%M:%S")}
mkdir -p "${LOGS_SUBDIR}"
LOG_NAME="${LOGS_SUBDIR}/test.log"
TEST_RESULTS="${LOGS_SUBDIR}/results.log"
BASE_VALS="--values=divingbell/values.yaml"
SYSCTL_KEY1=net.ipv4.conf.all.log_martians
SYSCTL_VAL1_DEFAULT=1
SYSCTL_KEY2=net.ipv4.conf.all.secure_redirects
SYSCTL_VAL2_DEFAULT=1
SYSCTL_KEY3=net.ipv4.conf.all.accept_redirects
SYSCTL_VAL3_DEFAULT=0
SYSCTL_KEY4=net/ipv6/conf/all/accept_redirects
SYSCTL_VAL4_DEFAULT=0
MOUNTS_SYSTEMD=/${NAME}
MOUNTS_PATH1=${MOUNTS_SYSTEMD}1
MOUNTS_PATH2=${MOUNTS_SYSTEMD}2
MOUNTS_PATH3=${MOUNTS_SYSTEMD}3
ETHTOOL_KEY2=tx-tcp-segmentation
ETHTOOL_VAL2_DEFAULT=on
ETHTOOL_KEY3=tx-tcp6-segmentation
# Not all NIC hardware has enough ethtool tunables available
ETHTOOL_KEY3_BACKUP=''
ETHTOOL_VAL3_DEFAULT=on
ETHTOOL_KEY4=tx-nocache-copy
ETHTOOL_VAL4_DEFAULT=off
ETHTOOL_KEY5=tx-checksum-ip-generic
ETHTOOL_KEY5_BACKUP=tx-scatter-gather
ETHTOOL_VAL5_DEFAULT=on
USERNAME1=userone
USERNAME1_SUDO=true
USERNAME1_SSHKEY1="ssh-rsa abc123 comment"
USERNAME2=usertwo
USERNAME2_SUDO=false
USERNAME2_SSHKEY1="ssh-rsa xyz456 comment"
USERNAME2_SSHKEY2="ssh-rsa qwe789 comment"
USERNAME2_SSHKEY3="ssh-rsa rfv000 comment"
USERNAME2_CRYPT_PASSWD='$6$AF.NLpphOJjMVTYC$GD6wyUTy9vIgatoMbtTDYcVtEJqh/Mrx3BRetVstMsNodSyn3ZFIZOMRePpRpGbFArnAxgkL1PtQxsZHCgtFn/'
USERNAME3=userthree
USERNAME3_SUDO=true
USERNAME4=userfour
USERNAME4_SUDO=false
APT_PACKAGE1=python-pbr
APT_VERSION1=1.8.0-4ubuntu1
APT_PACKAGE2=mysql-server
APT_PACKAGE3=python-simplejson
APT_VERSION3=3.8.1-1ubuntu2
APT_PACKAGE4=less
APT_PACKAGE5=python-setuptools
APT_PACKAGE6=telnetd
EXEC_DIR=/var/${NAME}/exec
# this used in test_overrides to check amount of daemonsets defined
EXPECTED_NUMBER_OF_DAEMONSETS=17
type lshw || apt -y install lshw
type apparmor_parser || apt -y install apparmor
nic_info="$(lshw -class network)"
physical_nic=''
IFS=$'\n'
for line in ${nic_info}; do
if [[ ${line} = *'physical id:'* ]]; then
physical_nic=true
fi
if [ "${physical_nic}" = 'true' ] && [[ ${line} = *'logical name'* ]]; then
DEVICE="$(echo "${line}" | cut -d':' -f2 | tr -d '[:space:]')"
echo "Found device: '${DEVICE}' to use for ethtool testing"
break
fi
done
[ -n "${DEVICE}" ] || (echo Could not find physical NIC for tesing; exit 1)
# Not all hardware has the same NIC tunables to use for testing
if [[ $(/sbin/ethtool -k "${DEVICE}" | grep "${ETHTOOL_KEY3}:") =~ .*fixed.* ]]; then
ETHTOOL_KEY3="${ETHTOOL_KEY3_BACKUP}"
fi
if [[ $(/sbin/ethtool -k "${DEVICE}" | grep "${ETHTOOL_KEY5}:") =~ .*fixed.* ]]; then
ETHTOOL_KEY5="${ETHTOOL_KEY5_BACKUP}"
fi
exec >& >(while read line; do echo "${line}" | sudo tee -a ${LOG_NAME}; done)
set -x
purge_containers(){
local chart_status="$(helm list ${NAME})"
if [ -n "${chart_status}" ]; then
helm delete --purge ${NAME}
fi
}
__set_systemd_name(){
if [ "${2}" = 'mount' ]; then
SYSTEMD_NAME="$(systemd-escape -p --suffix=mount "${1}")"
else
SYSTEMD_NAME="$(systemd-escape -p --suffix=service "${1}")"
fi
}
_teardown_systemd(){
__set_systemd_name "${1}" "${2}"
sudo systemctl stop "${SYSTEMD_NAME}" >& /dev/null || true
sudo systemctl disable "${SYSTEMD_NAME}" >& /dev/null || true
sudo rm "/etc/systemd/system/${SYSTEMD_NAME}" >& /dev/null || true
}
clean_persistent_files(){
sudo rm -r /var/${NAME} >& /dev/null || true
sudo rm -r /etc/sysctl.d/60-${NAME}-* >& /dev/null || true
sudo rm -r /etc/security/limits.d/60-${NAME}-* >& /dev/null || true
sudo rm -r /etc/apparmor.d/${NAME}-* >& /dev/null || true
_teardown_systemd ${MOUNTS_PATH1} mount
_teardown_systemd ${MOUNTS_PATH2} mount
_teardown_systemd ${MOUNTS_PATH3} mount
sudo systemctl daemon-reload
}
_write_sysctl(){
sudo /sbin/sysctl -w ${1}=${2}
}
_write_ethtool(){
local cur_val
if [ -z "${2}" ]; then
return
fi
cur_val="$(/sbin/ethtool -k ${1} |
grep "${2}:" | cut -d':' -f2 | cut -d' ' -f2)"
if [ "${cur_val}" != "${3}" ]; then
sudo /sbin/ethtool -K ${1} ${2} ${3} || true
fi
}
_reset_account(){
if [ -n "$1" ]; then
sudo deluser $1 >& /dev/null || true
sudo rm -r /home/$1 >& /dev/null || true
sudo rm /etc/sudoers.d/*$1* >& /dev/null || true
fi
}
init_default_state(){
purge_containers
clean_persistent_files
# set sysctl original vals
_write_sysctl ${SYSCTL_KEY1} ${SYSCTL_VAL1_DEFAULT}
_write_sysctl ${SYSCTL_KEY2} ${SYSCTL_VAL2_DEFAULT}
_write_sysctl ${SYSCTL_KEY3} ${SYSCTL_VAL3_DEFAULT}
_write_sysctl ${SYSCTL_KEY4} ${SYSCTL_VAL4_DEFAULT}
# set ethtool original vals
_write_ethtool ${DEVICE} ${ETHTOOL_KEY2} ${ETHTOOL_VAL2_DEFAULT}
_write_ethtool ${DEVICE} "${ETHTOOL_KEY3}" ${ETHTOOL_VAL3_DEFAULT}
_write_ethtool ${DEVICE} ${ETHTOOL_KEY4} ${ETHTOOL_VAL4_DEFAULT}
_write_ethtool ${DEVICE} ${ETHTOOL_KEY5} ${ETHTOOL_VAL5_DEFAULT}
# Remove any created accounts, SSH keys
_reset_account ${USERNAME1}
_reset_account ${USERNAME2}
_reset_account ${USERNAME3}
_reset_account ${USERNAME4}
}
install(){
purge_containers
helm install --name="${NAME}" --debug "${NAME}" --namespace="${NAME}" "$@"
}
upgrade(){
helm upgrade --name="${NAME}" --debug "${NAME}" --namespace="${NAME}" "$@"
}
dry_run(){
helm install --name="${NAME}" --dry-run --debug "${NAME}" --namespace="${NAME}" "$@"
}
get_container_status(){
local deployment="${1}"
local log_connect_timeout=60
local log_connect_sleep_interval=2
local wait_time=0
while : ; do
container="$(kubectl get pods --namespace="${NAME}" | grep ${NAME}-${deployment} | grep -v Terminating | cut -d' ' -f1)"
kubectl logs "${container}" --namespace="${NAME}" > /dev/null && break || \
echo "Waiting for container logs..." && \
wait_time=$((${wait_time} + ${log_connect_sleep_interval})) && \
sleep ${log_connect_sleep_interval}
if [ ${wait_time} -ge ${log_connect_timeout} ]; then
echo "Hit timeout while waiting for container logs to become available."
exit 1
fi
done
local container_runtime_timeout=210
local container_runtime_sleep_interval=5
wait_time=0
while : ; do
CLOGS="$(kubectl logs --namespace="${NAME}" "${container}" 2>&1)"
local status="$(echo "${CLOGS}" | tail -1)"
if [[ $(echo -e ${status} | tr -d '[:cntrl:]') = *ERROR* ]] ||
[[ $(echo -e ${status} | tr -d '[:cntrl:]') = *TRACE* ]]; then
if [ "${2}" = 'expect_failure' ]; then
echo 'Pod exited as expected'
break
else
echo 'Expected pod to complete successfully, but pod reported errors'
echo 'pod logs:'
echo "${CLOGS}"
exit 1
fi
elif [[ $(echo -e ${status} | tr -d '[:cntrl:]') = *'INFO Putting the daemon to sleep.'* ]] ||
[[ $(echo -e ${status} | tr -d '[:cntrl:]') = *'DEBUG + exit 0'* ]]; then
if [ "${2}" = 'expect_failure' ]; then
echo 'Expected pod to die with error, but pod completed successfully'
echo 'pod logs:'
echo "${CLOGS}"
exit 1
else
echo 'Pod completed without errors.'
break
fi
else
wait_time=$((${wait_time} + ${container_runtime_sleep_interval}))
sleep ${container_runtime_sleep_interval}
fi
if [ ${wait_time} -ge ${container_runtime_timeout} ]; then
echo 'Hit timeout while waiting for container to complete work.'
break
fi
done
}
_test_sysctl_default(){
test "$(/sbin/sysctl "${1}" | cut -d'=' -f2 | tr -d '[:space:]')" = "${2}"
}
_test_sysctl_value(){
_test_sysctl_default "${1}" "${2}"
local key="${1//\//.}"
test "$(cat /etc/sysctl.d/60-${NAME}-${key}.conf)" = "${key}=${2}"
}
_test_exec_match(){
expected_result="$1"
exec_testfile="$2"
testID="$3"
if [[ $expected_result != $(cat $exec_testfile) ]]; then
echo "[FAIL] exec $testID failed. Expected:"
echo $expected_result
echo but got:
echo $(cat $exec_testfile)
exit 1
fi
rm $exec_testfile
}
_test_exec_count(){
script_location="${1}"
script_name="${2}"
script_expected_run_count="${3}"
script_run_count=$(cat "${script_location}" | wc -l)
if [[ ${script_run_count} -ne ${script_expected_run_count} ]]; then
echo "[FAIL] Expected '${script_name}' to run '${script_expected_run_count}' times, but instead it ran '$script_run_count' times"
exit 1
fi
}
_test_clog_msg(){
[[ $CLOGS = *${1}* ]] ||
(echo "Did not find expected string: '${1}'"
echo "in container logs:"
echo "${CLOGS}"
exit 1)
}
alias install_base="install ${BASE_VALS}"
alias dry_run_base="dry_run ${BASE_VALS}"
shopt -s expand_aliases
test_sysctl(){
# Test the first set of values
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
local val1=0
local val2=1
local val3=0
local val4=0
echo "conf:
sysctl:
$SYSCTL_KEY1: $val1
$SYSCTL_KEY2: $val2
$SYSCTL_KEY3: $val3
$SYSCTL_KEY4: $val4" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status sysctl
_test_sysctl_value $SYSCTL_KEY1 $val1
_test_sysctl_value $SYSCTL_KEY2 $val2
_test_sysctl_value $SYSCTL_KEY3 $val3
_test_sysctl_value $SYSCTL_KEY4 $val4
echo '[SUCCESS] sysctl test1 passed successfully' >> "${TEST_RESULTS}"
# Test an updated set of values
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
val1=1
val2=0
val3=1
val4=1
echo "conf:
sysctl:
$SYSCTL_KEY1: $val1
$SYSCTL_KEY2: $val2
$SYSCTL_KEY3: $val3
$SYSCTL_KEY4: $val4" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status sysctl
_test_sysctl_value $SYSCTL_KEY1 $val1
_test_sysctl_value $SYSCTL_KEY2 $val2
_test_sysctl_value $SYSCTL_KEY3 $val3
_test_sysctl_value $SYSCTL_KEY4 $val4
echo '[SUCCESS] sysctl test2 passed successfully' >> "${TEST_RESULTS}"
# Test revert/rollback functionality
install_base
get_container_status sysctl
_test_sysctl_default $SYSCTL_KEY1 $SYSCTL_VAL1_DEFAULT
_test_sysctl_default $SYSCTL_KEY2 $SYSCTL_VAL2_DEFAULT
_test_sysctl_default $SYSCTL_KEY3 $SYSCTL_VAL3_DEFAULT
_test_sysctl_default $SYSCTL_KEY4 $SYSCTL_VAL4_DEFAULT
echo '[SUCCESS] sysctl test3 passed successfully' >> "${TEST_RESULTS}"
# Test invalid key
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid1.yaml
echo "conf:
sysctl:
this.is.a.bogus.key: 1" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status sysctl expect_failure
_test_clog_msg 'sysctl: cannot stat /proc/sys/this/is/a/bogus/key: No such file or directory'
echo '[SUCCESS] sysctl test4 passed successfully' >> "${TEST_RESULTS}"
# Test invalid val
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid2.yaml
echo "conf:
sysctl:
$SYSCTL_KEY1: bogus" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
# Sysctl does not report a non-zero exit code for this failure condition per
# https://bugzilla.redhat.com/show_bug.cgi?id=1264080
get_container_status sysctl
_test_clog_msg 'sysctl: setting key "net.ipv4.conf.all.log_martians": Invalid argument'
echo '[SUCCESS] sysctl test5 passed successfully' >> "${TEST_RESULTS}"
}
_test_limits_value(){
local limit=${1}
local domain=${2}
local type=${3}
local item=${4}
local value=${5}
test "$(cat /etc/security/limits.d/60-${NAME}-${limit}.conf)" = \
"$domain $type $item $value"
}
test_limits(){
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}.yaml
echo "conf:
limits:
limit1:
domain: root
type: hard
item: core
value: 0
limit2:
domain: '0:'
type: soft
item: nofile
value: 101" > "${overrides_yaml}"
echo $(cat ${overrides_yaml})
install_base "--values=${overrides_yaml}"
get_container_status limits
_test_limits_value limit1 root hard core 0
_test_limits_value limit2 '0:' soft nofile 101
echo "[SUCCESS] test range loop for limits passed successfully" >> "${TEST_RESULTS}"
}
_test_perm_value(){
local file=${1}
local owner=${2}
local group=${3}
local perm=${4}
local r_owner="$(stat -c %U ${file})"
local r_group="$(stat -c %G ${file})"
local r_perm="$(stat -c %a ${file})"
[ "${perm}"=="${r_perm}" ] && echo "+" || (echo "File ${file} permissions ${r_perm} but expected ${perm}"; exit 1)
[ "${owner}"=="${r_owner}" ] && echo "+" || (echo "File ${file} owner ${r_owner} but expected ${owner}"; exit 1)
[ "${group}"=="${r_group}" ] && echo "+" || (echo "File ${file} group ${r_group} but expected ${group}"; exit 1)
}
_perm_init_one(){
local file=${1}
local user=${file##*.}
useradd ${user} -U
chmod 777 ${file}
chown ${user}:${user} ${file}
echo ${file}
}
_make_p_temp(){
echo $(mktemp "${TMPDIR:-/tmp}/${0##*/}.XXXXXX")
}
_perm_init(){
# global vars!
p_test_file1=$(_perm_init_one $(_make_p_temp))
p_test_file2=$(_perm_init_one $(_make_p_temp))
}
_perm_teardown_one(){
local file=${1}
local user=${file##*.}
deluser ${user} -q
rm -f ${file}
}
_perm_teardown(){
# global vars!
_perm_teardown_one ${p_test_file1}
unset p_test_file1
_perm_teardown_one ${p_test_file2}
unset p_test_file2
}
test_perm(){
_perm_init
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}.yaml
echo "conf:
perm:
paths:
-
path: ${p_test_file1}
owner: 'root'
group: 'shadow'
permissions: '0640'
-
path: ${p_test_file2}
owner: 'root'
group: 'shadow'
permissions: '0640'" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status perm
_test_perm_value ${p_test_file1} root shadow 640
_test_perm_value ${p_test_file2} root shadow 640
echo "[SUCCESS] Positive test for perm passed successfully" >> "${TEST_RESULTS}"
echo "conf:
perm:
paths:
-
path: ${p_test_file1}
owner: 'root'
group: 'shadow'
permissions: '0640'" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status perm
_test_perm_value ${p_test_file1} root shadow 640
_test_perm_value ${p_test_file2} ${p_test_file2##*.} ${p_test_file2##*.} 777
echo "[SUCCESS] Backup test for perm passed successfully" >> "${TEST_RESULTS}"
# Test invalid rerun_interval (too short)
echo "conf:
perm:
rerun_interval: 30
paths:
-
path: ${p_test_file1}
owner: 'root'
group: 'shadow'
permissions: '0640'" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .rerun_interval. Got' || \
(echo "[FAIL] perm test invalid rerun_interval value did not receive expected 'BAD .rerun_interval. Got' error" && exit 1)
echo '[SUCCESS] perm test invalid rerun_interval passed successfully' >> "${TEST_RESULTS}"
# Test invalid rerun_interval combination
echo "conf:
perm:
rerun_interval: 60
rerun_policy: once_successfully
paths:
-
path: ${p_test_file1}
owner: 'root'
group: 'shadow'
permissions: '0640'" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD COMBINATION' || \
(echo "[FAIL] perm invalid rerun_interval combination did not receive expected 'BAD COMBINATION' error" && exit 1)
echo '[SUCCESS] perm invalid rerun_interval combination passed successfully' >> "${TEST_RESULTS}"
# test rerun_interval
echo "conf:
perm:
rerun_interval: 60
paths:
-
path: ${p_test_file1}
owner: 'root'
group: 'shadow'
permissions: '0640'" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status perm
sleep 72
get_container_status perm
_test_perm_value ${p_test_file1} root shadow 640
echo '[SUCCESS] perm rerun_interval passed successfully' >> "${TEST_RESULTS}"
_perm_teardown
}
_test_if_mounted_positive(){
mountpoint "${1}" || (echo "Expect ${1} to be mounted, but was not"; exit 1)
df -h | grep "${1}" | grep "${2}" ||
(echo "Did not find expected mount size of ${2} in mount table"; exit 1)
__set_systemd_name "${1}" mount
systemctl is-enabled "${SYSTEMD_NAME}" ||
(echo "Expect ${SYSTEMD_NAME} to be flagged to start on boot, but is not"
exit 1)
}
_test_if_mounted_negative(){
mountpoint "${1}" &&
(echo "Expect ${1} not to be mounted, but was"
exit 1) || true
__set_systemd_name "${1}" mount
systemctl is-enabled "${SYSTEMD_NAME}" &&
(echo "Expect ${SYSTEMD_NAME} not to be flagged to start on boot, but was"
exit 1) || true
}
test_mounts(){
# Test the first set of values
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
local mount_size=32M
echo "conf:
mounts:
mnt:
mnt_tgt: ${MOUNTS_PATH1}
device: tmpfs
type: tmpfs
options: 'defaults,noatime,nosuid,nodev,noexec,mode=1777,size=${mount_size}'
mnt2:
mnt_tgt: ${MOUNTS_PATH2}
device: tmpfs
type: tmpfs
options: 'defaults,noatime,nosuid,nodev,noexec,mode=1777,size=${mount_size}'
mnt3:
mnt_tgt: ${MOUNTS_PATH3}
device: tmpfs
type: tmpfs
options: 'defaults,noatime,nosuid,nodev,noexec,mode=1777,size=${mount_size}'
before: ntp.service
after: dbus.service" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status mounts
_test_if_mounted_positive ${MOUNTS_PATH1} ${mount_size}
_test_if_mounted_positive ${MOUNTS_PATH2} ${mount_size}
_test_if_mounted_positive ${MOUNTS_PATH3} ${mount_size}
echo '[SUCCESS] mounts test1 passed successfully' >> "${TEST_RESULTS}"
# Test an updated set of values
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
mount_size=30M
echo "conf:
mounts:
mnt:
mnt_tgt: ${MOUNTS_PATH1}
device: tmpfs
type: tmpfs
options: 'defaults,noatime,nosuid,nodev,noexec,mode=1777,size=${mount_size}'
mnt2:
mnt_tgt: ${MOUNTS_PATH2}
device: tmpfs
type: tmpfs
options: 'defaults,noatime,nosuid,nodev,noexec,mode=1777,size=${mount_size}'
mnt3:
mnt_tgt: ${MOUNTS_PATH3}
device: tmpfs
type: tmpfs
options: 'defaults,noatime,nosuid,nodev,noexec,mode=1777,size=${mount_size}'
before: ntp.service
after: dbus.service" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status mounts
_test_if_mounted_positive ${MOUNTS_PATH1} ${mount_size}
_test_if_mounted_positive ${MOUNTS_PATH2} ${mount_size}
_test_if_mounted_positive ${MOUNTS_PATH3} ${mount_size}
echo '[SUCCESS] mounts test2 passed successfully' >> "${TEST_RESULTS}"
# Test revert/rollback functionality
install_base
get_container_status mounts
_test_if_mounted_negative ${MOUNTS_PATH1}
_test_if_mounted_negative ${MOUNTS_PATH2}
_test_if_mounted_negative ${MOUNTS_PATH3}
echo '[SUCCESS] mounts test3 passed successfully' >> "${TEST_RESULTS}"
# Test invalid mount
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid1.yaml
echo "conf:
mounts:
mnt:
mnt_tgt: '${MOUNTS_PATH1}'
device: '/dev/bogus'
type: 'bogus'
options: 'defaults'" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status mounts expect_failure # systemd has long 3 min timeout
__set_systemd_name "${MOUNTS_PATH1}" mount
_test_clog_msg "${SYSTEMD_NAME} failed."
echo '[SUCCESS] mounts test4 passed successfully' >> "${TEST_RESULTS}"
}
_test_ethtool_value(){
if [ -z "${1}" ]; then
return
fi
test "$(/sbin/ethtool -k ${DEVICE} |
grep "${1}:" | cut -d':' -f2 | tr -d '[:space:]')" = "${2}"
}
test_ethtool(){
# Test the first set of values
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
local val2=on
local val3=off
[ -n "${ETHTOOL_KEY3}" ] && local line2_1="${ETHTOOL_KEY3}: $val3"
local val4=off
echo "conf:
ethtool:
${DEVICE}:
$ETHTOOL_KEY2: $val2
$line2_1
$ETHTOOL_KEY4: $val4" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status ethtool
_test_ethtool_value $ETHTOOL_KEY2 $val2
_test_ethtool_value "$ETHTOOL_KEY3" $val3
_test_ethtool_value $ETHTOOL_KEY4 $val4
echo '[SUCCESS] ethtool test1 passed successfully' >> "${TEST_RESULTS}"
# Test an updated set of values
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
val2=off
val3=on
[ -n "${ETHTOOL_KEY3}" ] && local line2_2="${ETHTOOL_KEY3}: $val3"
val4=on
echo "conf:
ethtool:
${DEVICE}:
$ETHTOOL_KEY2: $val2
$line2_2
$ETHTOOL_KEY4: $val4" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status ethtool
_test_ethtool_value $ETHTOOL_KEY2 $val2
_test_ethtool_value "$ETHTOOL_KEY3" $val3
_test_ethtool_value $ETHTOOL_KEY4 $val4
echo '[SUCCESS] ethtool test2 passed successfully' >> "${TEST_RESULTS}"
# Test revert/rollback functionality
install_base
get_container_status ethtool
_test_ethtool_value $ETHTOOL_KEY2 $ETHTOOL_VAL2_DEFAULT
_test_ethtool_value "$ETHTOOL_KEY3" $ETHTOOL_VAL3_DEFAULT
_test_ethtool_value $ETHTOOL_KEY4 $ETHTOOL_VAL4_DEFAULT
echo '[SUCCESS] ethtool test3 passed successfully' >> "${TEST_RESULTS}"
# Test invalid key
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid1.yaml
echo "conf:
ethtool:
${DEVICE}:
this-is-a-bogus-key: $val2" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status ethtool expect_failure
_test_clog_msg "Could not find requested param this-is-a-bogus-key for ${DEVICE}"
echo '[SUCCESS] ethtool test4 passed successfully' >> "${TEST_RESULTS}"
# Test invalid val
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid2.yaml
echo "conf:
ethtool:
${DEVICE}:
$ETHTOOL_KEY2: bogus" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status ethtool expect_failure
_test_clog_msg "Expected 'on' or 'off', got 'bogus'"
echo '[SUCCESS] ethtool test5 passed successfully' >> "${TEST_RESULTS}"
# Test fixed (unchangeable) ethtool param
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid3.yaml
echo "conf:
ethtool:
${DEVICE}:
hw-tc-offload: on" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status ethtool expect_failure
_test_clog_msg "does not permit changing the 'hw-tc-offload' setting"
echo '[SUCCESS] ethtool test6 passed successfully' >> "${TEST_RESULTS}"
# Test ethtool settings conflict
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid4.yaml
echo "conf:
ethtool:
${DEVICE}:
${ETHTOOL_KEY2}: on
${ETHTOOL_KEY5}: off" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status ethtool expect_failure
_test_clog_msg 'There is a conflict between settings chosen for this device.'
echo '[SUCCESS] ethtool test7 passed successfully' >> "${TEST_RESULTS}"
}
_test_user_enabled(){
username=$1
user_enabled=$2
if [ "${user_enabled}" = "true" ]; then
# verify the user is there and not set to expire
getent passwd $username >& /dev/null
test "$(chage -l ${username} | grep 'Account expires' | cut -d':' -f2 |
tr -d '[:space:]')" = "never"
else
# Verify user is not non-expiring
getent passwd $username >& /dev/null
test "$(chage -l ${username} | grep 'Account expires' | cut -d':' -f2 |
tr -d '[:space:]')" != "never"
fi
}
_test_user_purged(){
username=$1
# Verify user is no longer defined
getent passwd $username >& /dev/null && \
echo "Error: User '$username' exists, but was expected it to be purged" && \
return 1
if [ -d /home/$username ]; then
echo "Error: User '$username' home dir exists; expected it to be purged"
return 1
fi
}
_test_sudo_enabled(){
username=$1
sudo_enable=$2
sudoers_file=/etc/sudoers.d/*$username*
if [ "${sudo_enable}" = "true" ]; then
test -f $sudoers_file
else
test ! -f $sudoers_file
fi
}
_test_ssh_keys(){
username=$1
sshkey=$2
ssh_file=/home/$username/.ssh/authorized_keys
if [ "$sshkey" = "false" ]; then
test ! -f "${ssh_file}"
else
grep "$sshkey" "${ssh_file}"
fi
}
_test_user_passwd(){
username=$1
crypt_passwd="$2"
if [ "$crypt_passwd" != "$(getent shadow $username | cut -d':' -f2)" ]; then
echo "Error: User '$username' passwd did not match expected val '$crypt_passwd'"
return 1
fi
}
test_uamlite(){
# Test the first set of values
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
echo "conf:
uamlite:
users:
- user_name: ${USERNAME1}
user_sudo: ${USERNAME1_SUDO}
user_sshkeys:
- ${USERNAME1_SSHKEY1}
- user_name: ${USERNAME2}
user_sudo: ${USERNAME2_SUDO}
user_crypt_passwd: ${USERNAME2_CRYPT_PASSWD}
user_sshkeys:
- ${USERNAME2_SSHKEY1}
- ${USERNAME2_SSHKEY2}
- ${USERNAME2_SSHKEY3}
- user_name: ${USERNAME3}
user_sudo: ${USERNAME3_SUDO}
- user_name: ${USERNAME4}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status uamlite
_test_user_enabled ${USERNAME1} true
_test_sudo_enabled ${USERNAME1} ${USERNAME1_SUDO}
_test_ssh_keys ${USERNAME1} "${USERNAME1_SSHKEY1}"
_test_user_passwd ${USERNAME1} '*'
_test_user_enabled ${USERNAME2} true
_test_sudo_enabled ${USERNAME2} ${USERNAME2_SUDO}
_test_ssh_keys ${USERNAME2} "${USERNAME2_SSHKEY1}"
_test_ssh_keys ${USERNAME2} "${USERNAME2_SSHKEY2}"
_test_ssh_keys ${USERNAME2} "${USERNAME2_SSHKEY3}"
_test_user_passwd ${USERNAME2} ${USERNAME2_CRYPT_PASSWD}
_test_user_enabled ${USERNAME3} true
_test_sudo_enabled ${USERNAME3} ${USERNAME3_SUDO}
_test_ssh_keys ${USERNAME3} false
_test_user_passwd ${USERNAME3} '*'
_test_user_enabled ${USERNAME4} true
_test_sudo_enabled ${USERNAME4} ${USERNAME4_SUDO}
_test_ssh_keys ${USERNAME4} false
_test_user_passwd ${USERNAME4} '*'
echo '[SUCCESS] uamlite test1 passed successfully' >> "${TEST_RESULTS}"
# Test an updated set of values
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
uname1_sudo=false
uname2_sudo=true
uname3_sudo=false
echo "conf:
uamlite:
users:
- user_name: ${USERNAME1}
user_sudo: ${uname1_sudo}
- user_name: ${USERNAME2}
user_sudo: ${uname2_sudo}
user_sshkeys:
- ${USERNAME2_SSHKEY1}
- ${USERNAME2_SSHKEY2}
- user_name: ${USERNAME3}
user_sudo: ${uname3_sudo}
user_sshkeys:
- ${USERNAME1_SSHKEY1}
- ${USERNAME2_SSHKEY3}
- user_name: ${USERNAME4}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status uamlite
_test_user_enabled ${USERNAME1} true
_test_sudo_enabled ${USERNAME1} ${uname1_sudo}
_test_ssh_keys ${USERNAME1} false
_test_user_passwd ${USERNAME1} '*'
_test_user_enabled ${USERNAME2} true
_test_sudo_enabled ${USERNAME2} ${uname2_sudo}
_test_ssh_keys ${USERNAME2} "${USERNAME2_SSHKEY1}"
_test_ssh_keys ${USERNAME2} "${USERNAME2_SSHKEY2}"
_test_user_passwd ${USERNAME2} '*'
_test_user_enabled ${USERNAME3} true
_test_sudo_enabled ${USERNAME3} ${uname3_sudo}
_test_ssh_keys ${USERNAME3} "${USERNAME1_SSHKEY1}"
_test_ssh_keys ${USERNAME3} "${USERNAME2_SSHKEY3}"
_test_user_passwd ${USERNAME3} '*'
_test_user_enabled ${USERNAME4} true
_test_sudo_enabled ${USERNAME4} ${USERNAME4_SUDO}
_test_ssh_keys ${USERNAME4} false
_test_user_passwd ${USERNAME4} '*'
echo '[SUCCESS] uamlite test2 passed successfully' >> "${TEST_RESULTS}"
# Test revert/rollback functionality
install_base
get_container_status uamlite
_test_user_enabled ${USERNAME1} false
_test_sudo_enabled ${USERNAME1} false
_test_user_enabled ${USERNAME2} false
_test_sudo_enabled ${USERNAME2} false
_test_user_enabled ${USERNAME3} false
_test_sudo_enabled ${USERNAME3} false
_test_user_enabled ${USERNAME4} false
_test_sudo_enabled ${USERNAME4} false
echo '[SUCCESS] uamlite test3 passed successfully' >> "${TEST_RESULTS}"
# Test purge users flag
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set4.yaml
echo "conf:
uamlite:
purge_expired_users: true" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status uamlite
_test_user_purged ${USERNAME1}
_test_user_purged ${USERNAME2}
_test_user_purged ${USERNAME3}
_test_user_purged ${USERNAME4}
echo '[SUCCESS] uamlite test4 passed successfully' >> "${TEST_RESULTS}"
# Test invalid password
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set5.yaml
user2_crypt_passwd_invalid='plaintextPassword'
echo "conf:
uamlite:
users:
- user_name: ${USERNAME2}
user_crypt_passwd: ${user2_crypt_passwd_invalid}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD PASSWORD' || \
(echo "[FAIL] uamlite test5 did not receive expected 'BAD PASSWORD' error" && exit 1)
echo '[SUCCESS] uamlite test5 passed successfully' >> "${TEST_RESULTS}"
# Test invalid SSH key
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set6.yaml
user2_bad_sshkey='AAAAB3NzaC1yc2EAAAABIwAAAQEAklOUpkDHrfHY17SbrmT key-comment'
echo "conf:
uamlite:
users:
- user_name: ${USERNAME2}
user_sshkeys:
- ${USERNAME2_SSHKEY1}
- ${user2_bad_sshkey}
- ${USERNAME2_SSHKEY3}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD SSH KEY' || \
(echo "[FAIL] uamlite test6 did not receive expected 'BAD SSH KEY' error" && exit 1)
echo '[SUCCESS] uamlite test6 passed successfully' >> "${TEST_RESULTS}"
}
_test_apt_package_version(){
local pkg_name=$1
local pkg_ver=$2
if [ ${pkg_ver} = "none" ]; then
# Does not include residual-config
if [[ $(dpkg -l | grep ${pkg_name} | grep -v ^rc) ]]; then
echo "[FAIL] Package ${pkg_name} should not be installed" >> "${TEST_RESULTS}"
return 1
fi
elif [ ${pkg_ver} = "any" ]; then
if [[ ! $(dpkg -l | grep ${pkg_name}) ]]; then
echo "[FAIL] Package ${pkg_name} should be installed" >> "${TEST_RESULTS}"
return 1
fi
else
if [ $(dpkg -l | awk "/[[:space:]]${pkg_name}[[:space:]]/"'{print $3}') != "${pkg_ver}" ]; then
echo "[FAIL] Package ${pkg_name} should be of version ${pkg_ver}" >> "${TEST_RESULTS}"
return 1
fi
fi
}
test_apt(){
# Test the valid set of packages
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
echo "conf:
apt:
packages:
- name: $APT_PACKAGE1
version: $APT_VERSION1
- name: $APT_PACKAGE2" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apt
_test_apt_package_version $APT_PACKAGE1 $APT_VERSION1
_test_apt_package_version $APT_PACKAGE2 any
echo '[SUCCESS] apt test1 passed successfully' >> "${TEST_RESULTS}"
# Test removal of one package and install of one new package
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
echo "conf:
apt:
packages:
- name: $APT_PACKAGE2
debconf:
- question: mysql-server/root_password
question_type: password
answer: rootpw
- question: mysql-server/root_password_again
question_type: password
answer: rootpw
- name: $APT_PACKAGE3
version: $APT_VERSION3" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apt
_test_apt_package_version $APT_PACKAGE1 none
_test_apt_package_version $APT_PACKAGE2 any
# Each entry in passwords.dat contains question value in Name and Template
# field, so grepping root_password should return 4 lines
if [[ $(grep root_password /var/cache/debconf/passwords.dat | wc -l) != 4 ]]; then
echo "[FAIL] Package $APT_PACKAGE2 should have debconf values configured" >> "${TEST_RESULTS}"
return 1
fi
_test_apt_package_version $APT_PACKAGE3 $APT_VERSION3
echo '[SUCCESS] apt test2 passed successfully' >> "${TEST_RESULTS}"
# Test removal of all installed packages and install of one that already exists
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set3.yaml
echo "conf:
apt:
packages:
- name: $APT_PACKAGE4" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apt
_test_apt_package_version $APT_PACKAGE2 none
_test_apt_package_version $APT_PACKAGE3 none
echo '[SUCCESS] apt test3 passed successfully' >> "${TEST_RESULTS}"
# Test package not installed by divingbell not removed
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set4.yaml
echo "conf:
apt:
packages:
- name: $APT_PACKAGE5" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apt
_test_apt_package_version $APT_PACKAGE4 any # Should still be present
_test_apt_package_version $APT_PACKAGE5 any
echo '[SUCCESS] apt test4 passed successfully' >> "${TEST_RESULTS}"
# Test invalid package name
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid1.yaml
echo "conf:
apt:
packages:
- name: some-random-name
version: whatever" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apt expect_failure
_test_clog_msg 'E: Unable to locate package some-random-name'
echo '[SUCCESS] apt test5 passed successfully' >> "${TEST_RESULTS}"
# Test blacklistpkgs
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
echo "conf:
apt:
packages:
- name: $APT_PACKAGE6
blacklistpkgs:
- $APT_PACKAGE6" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apt
_test_apt_package_version $APT_PACKAGE6 none
echo '[SUCCESS] apt test6 passed successfully' >> "${TEST_RESULTS}"
}
# test exec module
test_exec(){
# test script execution ordering, args, and env vars
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
echo 'conf:
exec:
030-script5.sh:
blocking_policy: foreground_halt_pod_on_failure
env:
env1: env1-val
env2: env2-val
env3: env3-val
args:
- arg1
- arg2
- arg3
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> exec_testfile
echo args: "$@" >> exec_testfile
echo env: "$env1 $env2 $env3" >> exec_testfile
005-script1.sh:
blocking_policy: foreground
data: |
#!/bin/bash
rm exec_testfile 2> /dev/null || true
echo script name: ${BASH_SOURCE} >> exec_testfile
015-script3.sh:
blocking_policy: foreground_halt_pod_on_failure
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> exec_testfile
008-script2.sh:
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> exec_testfile
025-script4.sh:
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> exec_testfile' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status exec
expected_result='script name: ./005-script1.sh
script name: ./008-script2.sh
script name: ./015-script3.sh
script name: ./025-script4.sh
script name: ./030-script5.sh
args: arg1 arg2 arg3
env: env1-val env2-val env3-val'
_test_exec_match "$expected_result" "${EXEC_DIR}/exec_testfile" "test1"
echo '[SUCCESS] exec test1 passed successfully' >> "${TEST_RESULTS}"
# Test blocking_policy
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
echo 'conf:
exec:
030-script5.sh:
blocking_policy: foreground_halt_pod_on_failure
env:
env1: env1-val
env2: env2-val
env3: env3-val
args:
- arg1
- arg2
- arg3
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> exec_testfile
echo args: "$@" >> exec_testfile
echo env: "$env1 $env2 $env3" >> exec_testfile
005-script1.sh:
blocking_policy: foreground
data: |
#!/bin/bash
rm exec_testfile 2> /dev/null || true
echo script name: ${BASH_SOURCE} >> exec_testfile
015-script3.sh:
blocking_policy: foreground_halt_pod_on_failure
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> exec_testfile
false
008-script2.sh:
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> exec_testfile
025-script4.sh:
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> exec_testfile' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status exec expect_failure
expected_result='script name: ./005-script1.sh
script name: ./008-script2.sh
script name: ./015-script3.sh'
_test_exec_match "$expected_result" "${EXEC_DIR}/exec_testfile" "test2"
echo '[SUCCESS] exec test2 passed successfully' >> "${TEST_RESULTS}"
# Test invalid rerun_policy
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set3.yaml
echo 'conf:
exec:
030-script5.sh:
rerun_policy: foo
data: |
#!/bin/bash
true' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .rerun_policy. FOR' || \
(echo "[FAIL] exec test3 did not receive expected 'BAD .rerun_policy. FOR' error" && exit 1)
echo '[SUCCESS] exec test3 passed successfully' >> "${TEST_RESULTS}"
# Test invalid blocking_policy
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set4.yaml
echo 'conf:
exec:
030-script5.sh:
blocking_policy: foo
data: |
#!/bin/bash
true' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .blocking_policy. FOR' || \
(echo "[FAIL] exec test4 did not receive expected 'BAD .blocking_policy. FOR' error" && exit 1)
echo '[SUCCESS] exec test4 passed successfully' >> "${TEST_RESULTS}"
# Test rerun_policies:
# 1. Unspecified
# 2. always
# 3. once_successfully, when script passes
# 4. once_successfully, when script fails
# 5. never
# first execution
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set5.yaml
echo 'conf:
exec:
001-script1.sh:
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> script1
002-script2.sh:
rerun_policy: always
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> script2
003-script3.sh:
rerun_policy: once_successfully
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> script3
004-script4.sh:
rerun_policy: once_successfully
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> script4
false
005-script5.sh:
rerun_policy: never
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> script5
env:
env3: env3-val
env1: env1-val
env2: env2-val
args:
- arg2
- arg1
- arg3
manifests:
daemonset_ethtool: false
daemonset_mounts: false
daemonset_uamlite: false
daemonset_sysctl: false
daemonset_limits: false
daemonset_apt: false
daemonset_perm: false' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status exec
# run several times with the same values and evaluate results
# (ensure no ordering issues cause hashing inconsistencies)
for i in $(seq 0 11); do
install_base "--values=${overrides_yaml}"
get_container_status exec
_test_exec_count "${EXEC_DIR}/script1" '001-script1.sh' $(($i + 2))
_test_exec_count "${EXEC_DIR}/script2" '002-script1.sh' $(($i + 2))
_test_exec_count "${EXEC_DIR}/script3" '003-script1.sh' '1'
_test_exec_count "${EXEC_DIR}/script4" '004-script1.sh' $(($i + 2))
_test_exec_count "${EXEC_DIR}/script5" '005-script1.sh' '1'
echo "[SUCCESS] exec test$(($i + 5)) passed successfully" >> "${TEST_RESULTS}"
done
# test timeout
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set17.yaml
echo 'conf:
exec:
011-timeout.sh:
timeout: 11
data: |
#!/bin/bash
sleep 60' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status exec
_test_clog_msg 'timeout waiting for'
echo '[SUCCESS] exec test17 passed successfully' >> "${TEST_RESULTS}"
# Test invalid timeout
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set18.yaml
echo 'conf:
exec:
011-timeout.sh:
timeout: infinite
data: |
#!/bin/bash
sleep 60' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .timeout. FOR' || \
(echo "[FAIL] exec test18 did not receive expected 'BAD .timeout. FOR' error" && exit 1)
echo '[SUCCESS] exec test18 passed successfully' >> "${TEST_RESULTS}"
# Test invalid rerun_interval (too short)
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set19.yaml
echo 'conf:
exec:
012-rerun-interval.sh:
rerun_interval: 30
data: |
#!/bin/bash
true' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .rerun_interval. FOR' || \
(echo "[FAIL] exec test19 did not receive expected 'BAD .rerun_interval. FOR' error" && exit 1)
echo '[SUCCESS] exec test19 passed successfully' >> "${TEST_RESULTS}"
# Test invalid retry_interval (too short)
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set20.yaml
echo 'conf:
exec:
012-retry-interval.sh:
retry_interval: 30
data: |
#!/bin/bash
true' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .retry_interval. FOR' || \
(echo "[FAIL] exec test20 did not receive expected 'BAD .retry_interval. FOR' error" && exit 1)
echo '[SUCCESS] exec test20 passed successfully' >> "${TEST_RESULTS}"
# Test invalid rerun_interval combination
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set21.yaml
echo 'conf:
exec:
012-rerun-interval.sh:
rerun_interval: 60
rerun_policy: once_successfully
data: |
#!/bin/bash
true' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD COMBINATION' || \
(echo "[FAIL] exec test21 did not receive expected 'BAD COMBINATION' error" && exit 1)
echo '[SUCCESS] exec test21 passed successfully' >> "${TEST_RESULTS}"
# Test invalid retry_interval combination
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set22.yaml
echo 'conf:
exec:
012-retry-interval.sh:
retry_interval: 60
rerun_policy: never
data: |
#!/bin/bash
true' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD COMBINATION' || \
(echo "[FAIL] exec test22 did not receive expected 'BAD COMBINATION' error" && exit 1)
echo '[SUCCESS] exec test22 passed successfully' >> "${TEST_RESULTS}"
# test rerun_interval
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set23.yaml
echo 'conf:
exec:
012-rerun-interval.sh:
rerun_interval: 60
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> exec_testfile' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status exec
sleep 72
get_container_status exec
expected_result='script name: ./012-rerun-interval.sh
script name: ./012-rerun-interval.sh'
_test_exec_match "$expected_result" "${EXEC_DIR}/exec_testfile" "test23"
echo '[SUCCESS] exec test23 passed successfully' >> "${TEST_RESULTS}"
# test retry_interval
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set24.yaml
echo 'conf:
exec:
012-retry-interval.sh:
retry_interval: 60
data: |
#!/bin/bash
echo script name: ${BASH_SOURCE} >> exec_testfile
false' > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status exec
sleep 72
get_container_status exec
expected_result='script name: ./012-retry-interval.sh
script name: ./012-retry-interval.sh'
_test_exec_match "$expected_result" "${EXEC_DIR}/exec_testfile" "test24"
echo '[SUCCESS] exec test24 passed successfully' >> "${TEST_RESULTS}"
}
# test daemonset value overrides for hosts and labels
test_overrides(){
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-dryrun.yaml
echo "conf:
sysctl:
net.ipv4.ip_forward: 1
net.ipv6.conf.all.forwarding: 1
overrides:
divingbell_sysctl:
labels:
- label:
key: compute_type
values:
- dpdk
- sriov
conf:
sysctl:
net.ipv4.ip_forward: 1
- label:
key: compute_type
values:
- special
conf:
sysctl:
net.ipv4.ip_forward: 1
- label:
key: compute_type
values:
- special
conf:
sysctl:
net.ipv4.ip_forward: 1
hosts:
- name: superhost
conf:
sysctl:
net.ipv4.ip_forward: 0
net.ipv6.conf.all.forwarding: 0
- name: helm1
conf:
sysctl:
net.ipv6.conf.all.forwarding: 0
- name: specialhost
conf:
sysctl:
net.ipv6.conf.all.forwarding: 1
divingbell_mounts:
labels:
- label:
key: blarg
values:
- soup
- chips
conf:
mounts:
mnt:
mnt_tgt: /mnt
device: tmpfs
type: tmpfs
options: 'defaults,noatime,nosuid,nodev,noexec,mode=1777,size=32M'
divingbell_ethtool:
hosts:
- name: ethtool-host
conf:
ethtool:
ens3:
hw-tc-offload: on
divingbell_bogus:
labels:
- label:
key: bogus
values:
- foo
- bar
conf:
bogus:
other_stuff: XYZ
- label:
key: bogus_label
values:
- bogus_value
conf:
bogus:
more_stuff: ABC
hosts:
- name: superhost2
conf:
bogus:
other_stuff: FOO
more_stuff: BAR" > "${overrides_yaml}"
tc_output="$(dry_run_base "--values=${overrides_yaml}")"
# Compare against expected number of generated daemonsets
daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)"
if [ "${daemonset_count}" != "${EXPECTED_NUMBER_OF_DAEMONSETS}" ]; then
echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}"
echo "Expected ${EXPECTED_NUMBER_OF_DAEMONSETS} daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
exit 1
else
echo '[SUCCESS] overrides test 1 passed successfully' >> "${TEST_RESULTS}"
fi
# TODO: Implement more robust tests that do not depend on match expression
# ordering.
# Verify generated affinity for another_label
echo "${tc_output}" | grep ' spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: another_label
operator: In
values:
- "another_value"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "superhost"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "helm1"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "specialhost"' &&
echo '[SUCCESS] overrides test 2 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 2 failed' && exit 1)
# Verify generated affinity for compute_type
echo "${tc_output}" | grep ' spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: compute_type
operator: In
values:
- "special"
- key: another_label
operator: NotIn
values:
- "another_value"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "superhost"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "helm1"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "specialhost"' &&
echo '[SUCCESS] overrides test 3 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 3 failed' && exit 1)
# Verify generated affinity for compute_type
echo "${tc_output}" | grep ' spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: compute_type
operator: In
values:
- "dpdk"
- "sriov"
- key: compute_type
operator: NotIn
values:
- "special"
- key: another_label
operator: NotIn
values:
- "another_value"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "superhost"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "helm1"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "specialhost"' &&
echo '[SUCCESS] overrides test 4 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 4 failed' && exit 1)
# Verify generated affinity for one of the daemonset hosts
echo "${tc_output}" | grep ' spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: blarg
operator: In
values:
- "soup"
- "chips"' &&
echo '[SUCCESS] overrides test 5 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 5 failed' && exit 1)
# Verify generated affinity for one of the daemonset defaults
echo "${tc_output}" | grep ' spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- "superhost"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "helm1"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "specialhost"
- key: compute_type
operator: NotIn
values:
- "dpdk"
- "sriov"
- key: compute_type
operator: NotIn
values:
- "special"
- key: another_label
operator: NotIn
values:
- "another_value"' &&
echo '[SUCCESS] overrides test 6 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 6 failed' && exit 1)
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-functional.yaml
key1_override_val=0
key2_non_override_val=0
echo "conf:
sysctl:
$SYSCTL_KEY1: 1
$SYSCTL_KEY2: $key2_non_override_val
overrides:
divingbell_sysctl:
hosts:
- name: $(hostname -f)
conf:
sysctl:
$SYSCTL_KEY1: $key1_override_val" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status sysctl
_test_sysctl_default $SYSCTL_KEY1 $key1_override_val
_test_sysctl_default $SYSCTL_KEY2 $key2_non_override_val
echo '[SUCCESS] overrides test 7 passed successfully' >> "${TEST_RESULTS}"
}
_test_apparmor_profile_added(){
local profile_file=$1
local profile_name=$2
local defaults_path='/var/divingbell/apparmor'
local persist_path='/etc/apparmor.d'
if [ ! -f "${defaults_path}/${profile_file}" ]; then
return 1
fi
if [ ! -L "${persist_path}/${profile_file}" ]; then
return 1
fi
profile_loaded=$(grep $profile_name /sys/kernel/security/apparmor/profiles || : )
if [ -z "$profile_loaded" ]; then
return 1
fi
return 0
}
_test_apparmor_profile_removed(){
local profile_file=$1
local profile_name=$2
local defaults_path='/var/divingbell/apparmor'
local persist_path='/etc/apparmor.d'
if [ -f "${defaults_path}/${profile_file}" ]; then
return 1
fi
if [ -L "${persist_path}/${profile_file}" ]; then
return 1
fi
profile_loaded=$(grep $profile_name /sys/kernel/security/apparmor/profiles || : )
if [ ! -z "$profile_loaded" ]; then
return 1
fi
reboot_message_present=$(grep $profile_file /var/run/reboot-required.pkgs || : )
if [ -z "$reboot_message_present" ]; then
return 1
fi
return 0
}
test_apparmor(){
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-apparmor.yaml
#Test1 - check new profile added and loaded
echo "conf:
apparmor:
profiles:
divingbell-profile-1: |
#include <tunables/global>
/usr/sbin/profile-1 {
#include <abstractions/apache2-common>
#include <abstractions/base>
#include <abstractions/nis>
capability dac_override,
capability dac_read_search,
capability net_bind_service,
capability setgid,
capability setuid,
/data/www/safe/* r,
deny /data/www/unsafe/* r,
}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apparmor
_test_apparmor_profile_added divingbell-profile-1 profile-1
echo '[SUCCESS] apparmor test1 passed successfully' >> "${TEST_RESULTS}"
#Test2 - check new profile added and loaded, profile-1 still exist
echo "conf:
apparmor:
profiles:
divingbell-profile-1: |
#include <tunables/global>
/usr/sbin/profile-1 {
#include <abstractions/apache2-common>
#include <abstractions/base>
#include <abstractions/nis>
capability dac_override,
capability dac_read_search,
capability net_bind_service,
capability setgid,
capability setuid,
/data/www/safe/* r,
deny /data/www/unsafe/* r,
}
divingbell-profile-2: |
#include <tunables/global>
/usr/sbin/profile-2 {
#include <abstractions/apache2-common>
#include <abstractions/base>
#include <abstractions/nis>
capability dac_override,
capability dac_read_search,
capability net_bind_service,
capability setgid,
capability setuid,
/data/www/safe/* r,
deny /data/www/unsafe/* r,
}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apparmor
_test_apparmor_profile_added divingbell-profile-1 profile-1
_test_apparmor_profile_added divingbell-profile-2 profile-2
echo '[SUCCESS] apparmor test2 passed successfully' >> "${TEST_RESULTS}"
#Test3 - check profile-2 removed, profile-1 still exist
echo "conf:
apparmor:
complain_mode: true
profiles:
divingbell-profile-1: |
#include <tunables/global>
/usr/sbin/profile-1 {
#include <abstractions/apache2-common>
#include <abstractions/base>
#include <abstractions/nis>
capability dac_override,
capability dac_read_search,
capability net_bind_service,
capability setgid,
capability setuid,
/data/www/safe/* r,
deny /data/www/unsafe/* r,
}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apparmor
_test_apparmor_profile_added divingbell-profile-1 profile-1
_test_apparmor_profile_removed divingbell-profile-2 profile-2
echo '[SUCCESS] apparmor test3 passed successfully' >> "${TEST_RESULTS}"
#Test4 - check for bad profile input
echo "conf:
apparmor:
profiles:
divingbell-profile-3: |
#include <tunables/global>
/usr/sbin/profile-3 {
bad data
}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apparmor expect_failure
_test_clog_msg 'AppArmor parser error for /etc/apparmor.d/divingbell-profile-3 in /etc/apparmor.d/divingbell-profile-3 at line 3: syntax error, unexpected TOK_ID, expecting TOK_MODE'
echo '[SUCCESS] apparmor test4 passed successfully' >> "${TEST_RESULTS}"
}
# initialization
init_default_state
# run tests
if [[ -z $SKIP_BASE_TESTS ]]; then
install_base
test_sysctl
test_limits
test_perm
test_mounts
test_ethtool
test_uamlite
test_apt
test_exec
test_apparmor
fi
purge_containers
test_overrides
# restore initial state
init_default_state
echo "All tests pass for ${NAME}"
View Git Blame Copy Permalink