Browse Source

Make kube-proxy liveness probe more cautious

This update makes it so list of services without endpoints detected on
the host must be static to cause failure.

This avoids race conditions for large deployments where new services are
being added over several minutes, and trigger probe failures.

Change-Id: Ie65c8613cb85bfdf61d41099540d3499ea1de817
Mark Burnett 6 months ago
parent
commit
eaeb3ae250
1 changed files with 17 additions and 5 deletions
  1. 17
    5
      charts/proxy/templates/bin/_liveness-probe.sh.tpl

+ 17
- 5
charts/proxy/templates/bin/_liveness-probe.sh.tpl View File

@@ -2,6 +2,8 @@
2 2
 
3 3
 set -e
4 4
 
5
+IPTS_DIR=/tmp/liveness
6
+
5 7
 FAILURE=0
6 8
 {{- if .Values.livenessProbe.whitelist }}
7 9
 WHITELIST='({{- join "|" .Values.livenessProbe.whitelist -}})'
@@ -15,12 +17,23 @@ if [[ $(echo -e "${REQUEST}" | socat - TCP4:localhost:10256 | grep -sc '200 OK')
15 17
     FAILURE=1
16 18
 fi
17 19
 
18
-if [[ $(iptables-save {{- if .Values.livenessProbe.whitelist }} | grep -Ev "${WHITELIST}" {{- end }} | grep -sc 'has no endpoints') -gt 0 ]]; then
19
-    echo Some non-whitelisted services have no endpoints:
20
-    iptables-save | grep 'has no endpoints'
21
-    FAILURE=1
20
+mkdir -p "${IPTS_DIR}"
21
+iptables-save {{- if .Values.livenessProbe.whitelist }} | grep -Ev "${WHITELIST}" {{- end }} | grep -s 'has no endpoints' | sort > "${IPTS_DIR}/current"
22
+
23
+if [[ $(wc -l "${IPTS_DIR}/current") -gt 0 ]]; then
24
+    if [[ "${IPTS_DIR}/previous" ]]; then
25
+        if cmp "${IPTS_DIR}/current" "${IPTS_DIR}/previous"; then
26
+            echo Some non-whitelisted services have no endpoints:
27
+            cat "${IPTS_DIR}/current"
28
+            FAILURE=1
29
+        else
30
+            echo Detected issues have changed.  Passing check:
31
+            diff "${IPTS_DIR}/previous" "${IPTS_DIR}/current"
32
+        fi
33
+    fi
22 34
 fi
23 35
 
36
+mv "${IPTS_DIR}/current" "${IPTS_DIR}/previous"
24 37
 
25 38
 IPTABLES_IPS=$(iptables-save | grep -E 'KUBE-SEP.*to-destination' | sed 's/.*to-destination \(.*\):.*/\1/' | sort -u)
26 39
 KUBECTL_IPS=$(kubectl get --all-namespaces -o json endpoints | jq -r '.items | arrays | .[] | objects | .subsets | arrays | .[] | objects | .addresses | arrays | .[] | objects | .ip' | sort -u)
@@ -31,7 +44,6 @@ if [[ $(comm -23 <(echo "${IPTABLES_IPS}") <(echo "${KUBECTL_IPS}")) ]]; then
31 44
     comm -23 <(echo "${IPTABLES_IPS}") <(echo "${KUBECTL_IPS}")
32 45
 fi
33 46
 
34
-
35 47
 if [[ "${FAILURE}" == "1" ]]; then
36 48
     exit 1
37 49
 fi

Loading…
Cancel
Save