Add readiness probe to divingbell-exec

Also add dist-upgrade verification.

Change-Id: I0716ee878e9a2fa9a557debe543996691c0540ce
Signed-off-by: Ruslan Aliev <raliev@mirantis.com>

divingbell/Chart.yaml

@@ -15,4 +15,4 @@
 apiVersion: v1
 description: divingbell
 name: divingbell
-version: 0.1.2
+version: 0.1.3

divingbell/templates/bin/_apt.sh.tpl

@@ -65,6 +65,7 @@ wait_for_dpkg_availability(){
 # First 5 lines are field descriptions
 wait_for_dpkg_availability
 load_package_list_with_versions $(dpkg -l | awk 'NR>5 {print $2"="$3}')
+declare -A package_info; while read -r name version; do package_info["$name"]="$version N/A N/A"; done < <(dpkg -l | awk '/^ii/ {gsub(/:[^ ]*/, "", $2); print $2, $3}')
 
 ################################################
 #Stage 2
@@ -262,6 +263,38 @@ dpkg --configure -a --force-confold,confdef
 {{- end }}
 {{- end }}
 
+######################################################
+#Stage 5
+#Verify that all package versions are correct and latest
+######################################################
+{{- if .Values.conf.apt.upgrade }}
+while read -r name version; do
+  if [[ "${package_info[$name]+exists}" ]]; then
+    package_info["$name"]="$(echo ${package_info[$name]} | awk -v val=$version '{$2 = val; print}')"
+  else
+    package_info["$name"]="N/A $version N/A"
+  fi
+done < <(dpkg -l | awk '/^ii/ {gsub(/:[^ ]*/, "", $2); print $2, $3}')
+
+while read -r name version; do
+  if [ -n "$version" ]; then
+    package_info["$name"]="$(echo ${package_info[$name]} | awk -v val=$version '{$3 = val; print}')"
+  fi
+done < <(apt-cache madison "${!package_info[@]}" | awk '!a[$1]++ {print $1 " " $3}')
+
+for pkg in "${!package_info[@]}"; do
+  read -r before after available < <(echo "${package_info[$pkg]}")
+  if [[ "$after" != "N/A" && "$after" != "$available" ]]; then
+    log.ERROR "Package $pkg was not properly installed/upgraded, installed version $after, available version $available"
+    exit 1
+  fi
+  if [[ "$before" != "$after" ]]; then
+    [[ "$before" == "N/A" && "$after" != "N/A" ]] && log.INFO "$pkg installed ver $after"
+    [[ "$before" != "N/A" && "$after" == "N/A" ]] && log.INFO "$pkg removed ver $before"
+    [[ "$before" != "N/A" && "$after" != "N/A" ]] && log.INFO "$pkg upgraded ver $before => $after"
+  fi
+done
+{{- end }}
 log.INFO 'Putting the daemon to sleep.'
 EOF
 

divingbell/templates/bin/_exec.sh.tpl

@@ -20,6 +20,8 @@
 
 set -e
 
+rm -rf /tmp/healthy
+
 cat <<'UNIQUE_EOF_9c341059-25a0-4725-9489-1789e255e381' > {{ .Values.conf.chroot_mnt_path | quote }}/tmp/exec_host_{{ .Chart.Version }}.sh
 {{ include "divingbell.shcommon" . }}
 
@@ -217,6 +219,10 @@ UNIQUE_EOF_9c341059-25a0-4725-9489-1789e255e381
 chmod 700 {{ .Values.conf.chroot_mnt_path | quote }}/tmp/exec_host_{{ .Chart.Version }}.sh
 
 while true; do
-  nsenter -t 1 -m -u -n -i -p /tmp/exec_host_{{ .Chart.Version }}.sh
+  if nsenter -t 1 -m -u -n -i -p /tmp/exec_host_{{ .Chart.Version }}.sh ; then
+    [ ! -f /tmp/healthy ] && touch /tmp/healthy
+  else
+    rm -rf /tmp/healthy
+  fi
   sleep {{ $exec_loop_sleep_interval }}
 done

divingbell/templates/daemonset-exec.yaml

@@ -14,6 +14,13 @@
 # limitations under the License.
 */}}
 
+{{- define "readinessProbeTemplate" }}
+  exec:
+    command:
+    - cat
+    - /tmp/healthy
+{{- end }}
+
 {{- define "divingbell.daemonset.exec" }}
   {{- $daemonset := index . 0 }}
   {{- $secretName := index . 1 }}
@@ -51,6 +58,7 @@ spec:
         imagePullPolicy: {{ .Values.images.pull_policy }}
 {{ tuple $envAll $envAll.Values.pod.resources.exec | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
 {{ dict "envAll" $envAll "application" "divingbell" "container" "exec" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 8 }}
+{{ dict "envAll" $envAll "component" "divingbell" "container" "exec" "type" "readiness" "probeTemplate" (include "readinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 8 }}
         command:
         - /tmp/{{ $daemonset }}.sh
         volumeMounts:

divingbell/values.yaml

@@ -267,9 +267,16 @@ pod:
         readiness:
           enabled: true
           params:
-            initialDelaySeconds: 5
-            periodSeconds: 5
-            failureThreshold: 120
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            failureThreshold: 1200
+      exec:
+        readiness:
+          enabled: true
+          params:
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            failureThreshold: 1200
 
 network_policy:
   divingbell:

tox.ini

@@ -5,8 +5,8 @@ envlist = docs
 
 [testenv]
 usedevelop = True
-whitelist_externals = rm
-passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY PBR_VERSION
+allowlist_externals = rm
+passenv = http_proxy,HTTP_PROXY,https_proxy,HTTPS_PROXY,no_proxy,NO_PROXY,PBR_VERSION
 setenv = VIRTUAL_ENV={envdir}
 install_command = pip install {opts} {packages}
 deps = -r{toxinidir}/test-requirements.txt