From e11888c1df5cffa1fcbe97b1e3371667380620f1 Mon Sep 17 00:00:00 2001 From: Thiago Brito Date: Sat, 31 Jul 2021 00:26:49 -0300 Subject: [PATCH] Enabling liveness probe for MariaDB In a DX scenario, after lock-unlock a controller the remaining MariaDB instance (lets say maria-server0) goes to a Non-Primary + Initializing State (non-operational). After that it remains searching for the now deleted pod (maria-server1) but using the old IP, the one before the restart. maria-server0 flags the old IP as delayed and suspect for eviction, however being a Non-Primary member it cannot in fact evict the old node and start looking for new members. Setting a LivenessProbe that detects nonoperational members and restart them fixes this, as the recreated pod starts looking for a cluster to join. Closes-Bug: #1938346 Signed-off-by: Thiago Brito Change-Id: I38d788f720cbd6bd13b6b6147db6f3d2a2ff9c92 --- ...de-of-mariadb-server-probe-parameters.patch | 18 +++++++++++++----- .../stx-openstack-helm/manifests/manifest.yaml | 2 ++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/openstack-helm-infra/files/0009-Enable-override-of-mariadb-server-probe-parameters.patch b/openstack-helm-infra/files/0009-Enable-override-of-mariadb-server-probe-parameters.patch index a2326039..89e642af 100644 --- a/openstack-helm-infra/files/0009-Enable-override-of-mariadb-server-probe-parameters.patch +++ b/openstack-helm-infra/files/0009-Enable-override-of-mariadb-server-probe-parameters.patch @@ -17,21 +17,22 @@ parameters and override the related probe pareameters. Signed-off-by: Hu, Yong Co-Authored-By: Zhipeng, Liu --- - mariadb/templates/statefulset.yaml | 3 ++- - mariadb/values.yaml | 7 +++++++ - 2 files changed, 9 insertions(+), 1 deletion(-) + mariadb/templates/statefulset.yaml | 3 +++- + mariadb/values.yaml | 20 +++++++++++++++ + 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mariadb/templates/statefulset.yaml b/mariadb/templates/statefulset.yaml index e31d4ac..83e135f 100644 --- a/mariadb/templates/statefulset.yaml +++ b/mariadb/templates/statefulset.yaml -@@ -187,7 +187,8 @@ spec: +@@ -187,7 +187,9 @@ spec: exec: command: - /tmp/stop.sh -{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "readiness" "probeTemplate" (include "mariadbReadinessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 -}} +{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "readiness" "probeTemplate" (include "mariadbReadinessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "startup" "probeTemplate" (include "mariadbReadinessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} ++{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "liveness" "probeTemplate" (include "mariadbReadinessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} volumeMounts: - name: pod-tmp mountPath: /tmp @@ -39,7 +40,7 @@ diff --git a/mariadb/values.yaml b/mariadb/values.yaml index 2d75f39..444bba3 100644 --- a/mariadb/values.yaml +++ b/mariadb/values.yaml -@@ -69,6 +69,13 @@ pod: +@@ -69,6 +69,20 @@ pod: initialDelaySeconds: 30 periodSeconds: 30 timeoutSeconds: 15 @@ -50,6 +51,13 @@ index 2d75f39..444bba3 100644 + initialDelaySeconds: 60 + periodSeconds: 60 + failureThreshold: 10 ++ liveness: ++ enabled: false ++ params: ++ initialDelaySeconds: 60 ++ periodSeconds: 30 ++ failureThreshold: 5 ++ timeoutSeconds: 15 security_context: server: pod: diff --git a/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml b/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml index 595843f4..06b91ef1 100644 --- a/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml +++ b/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml @@ -179,6 +179,8 @@ data: enabled: true startup: enabled: true + liveness: + enabled: true affinity: anti: type: