openstack/kolla-ansible
Code Issues Proposed changes

CI: Fail on fluentd log parsing errors

Browse Source 
Change-Id: Ie3963f5ed20f7fb61ef2e03f0cf12a4ea1c87c9c
This commit is contained in:
Michal Nasiadka 2020-07-22 13:50:17 +02:00
parent 0b1a59dd8c
commit ba54f8cdda
1 changed files with 17 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
tests/check-logs.sh
View File

@ -8,6 +8,7 @@ set -o pipefail
# Enable unbuffered output for Ansible in Jenkins. # Enable unbuffered output for Ansible in Jenkins.
export PYTHONUNBUFFERED=1 export PYTHONUNBUFFERED=1
declare -a fluentchecks=("got incomplete line before first line" "pattern not matched")
function check_openstack_log_file_for_level { function check_openstack_log_file_for_level {
# $1: file # $1: file
@ -22,6 +23,12 @@ function check_fluentd_log_file_for_level {
sudo egrep "\[$2\]:" $1 sudo egrep "\[$2\]:" $1
} }
function check_fluentd_log_file_for_content {
# $1: file
# $2: content
sudo egrep " $2 " $1
}
function check_docker_log_file_for_sigkill { function check_docker_log_file_for_sigkill {
sudo journalctl --no-pager -u ${CONTAINER_ENGINE}.service | grep "signal 9" sudo journalctl --no-pager -u ${CONTAINER_ENGINE}.service | grep "signal 9"
} }
@ -73,7 +80,6 @@ for level in CRITICAL ERROR WARNING; do
fi fi
done done
# check fluentd errors (we consider them critical) # check fluentd errors (we consider them critical)
fluentd_log_file=/var/log/kolla/fluentd/fluentd.log fluentd_log_file=/var/log/kolla/fluentd/fluentd.log
fluentd_error_summary_file=/tmp/logs/kolla/fluentd-error.log fluentd_error_summary_file=/tmp/logs/kolla/fluentd-error.log
@ -84,6 +90,16 @@ if check_fluentd_log_file_for_level $fluentd_log_file error >/dev/null; then
echo >> $fluentd_error_summary_file echo >> $fluentd_error_summary_file
fi fi
for string in "${fluentchecks[@]}"; do
fluentd_file=/tmp/logs/kolla/fluentd-errors.log
if check_fluentd_log_file_for_content $fluentd_log_file "$string" >/dev/null; then
any_critical=1
echo "$string" >> $fluentd_file
check_fluentd_log_file_for_content $fluentd_log_file "$string" >> $fluentd_file
echo >> $fluentd_file
fi
done
if check_docker_log_file_for_sigkill >/dev/null; then if check_docker_log_file_for_sigkill >/dev/null; then
any_critical=1 any_critical=1
echo "(critical) Found containers killed using signal 9 (SIGKILL) in docker logs." echo "(critical) Found containers killed using signal 9 (SIGKILL) in docker logs."