CI: Don't fail on expected critical log messages

There is a number of critical log messages that we see in CI from time
to time. While these should be fixed, let's not fail jobs unnecessarily.

This change introduces one expected critical message in
placement-api.log:

    Failed to fetch token data from identity server

Co-authored-by: Radosław Piliszek <radoslaw.piliszek@gmail.com>
Related-bug: #1847727
Change-Id: I92ad0be70ed05925612f0c709907ab62280326b8

tests/check-logs.sh

@@ -16,6 +16,25 @@ function check_file_for_level {
     sudo egrep " $2 " $1 | egrep -v "(logging_exception_prefix|rate_limit_except_level)"
 }
 
+function filter_out_expected_critical {
+    # $1: file
+    # Filter out expected critical log messages that we do not want to fail the
+    # job.
+
+    case $1 in
+    */placement-api.log)
+        # Sometimes we see this during upgrade when keystone is down.
+        grep -v "Failed to fetch token data from identity server"
+        ;;
+    *)
+        # We have to provide some pass-through consumer to avoid:
+        #   grep: write error: Broken pipe
+        # from check_file_for_level
+        cat
+        ;;
+    esac
+}
+
 any_critical=0
 for level in CRITICAL ERROR WARNING; do
     all_file=/tmp/logs/kolla/all-${level}.log
@@ -27,7 +46,9 @@ for level in CRITICAL ERROR WARNING; do
         if check_file_for_level $f $level >/dev/null; then
             any_matched=1
             if [[ $level = CRITICAL ]]; then
-                any_critical=1
+                if check_file_for_level $f $level | filter_out_expected_critical $f >/dev/null; then
+                    any_critical=1
+                fi
             fi
             echo $f >> $all_file
             check_file_for_level $f $level >> $all_file