Neutron: allow control of probes

This PS allows the probes in containers/pods to be tuned via values
overrides.

Depends-On: https://review.opendev.org/#/c/631597/

Change-Id: I439dce38a1b7df8c798f10f7fad406f9b0dfe3e6
Signed-off-by: Pete Birley <pete@port.direct>

neutron/templates/daemonset-dhcp-agent.yaml

@@ -14,6 +14,31 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- define "dhcpAgentReadinessProbeTemplate" }}
+exec:
+  command:
+    - python
+    - /tmp/health-probe.py
+    - --config-file
+    - /etc/neutron/neutron.conf
+    - --config-file
+    - /etc/neutron/dhcp_agent.ini
+    - --agent-queue-name
+    - dhcp_agent
+{{- end }}
+{{- define "dhcpAgentLivenessProbeTemplate" }}
+exec:
+  command:
+    - python
+    - /tmp/health-probe.py
+    - --config-file
+    - /etc/neutron/neutron.conf
+    - --config-file
+    - /etc/neutron/dhcp_agent.ini
+    - --agent-queue-name
+    - dhcp_agent
+{{- end }}
+
 {{- define "neutron.dhcp_agent.daemonset" }}
 {{- $daemonset := index . 0 }}
 {{- $configMapName := index . 1 }}
@@ -65,35 +90,8 @@ spec:
 {{ tuple $envAll "neutron_dhcp" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.dhcp | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "neutron" "container" "neutron_dhcp_agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
-          readinessProbe:
+{{ dict "envAll" $envAll "component" "dhcp_agent" "container" "dhcp_agent" "type" "readiness" "probeTemplate" (include "dhcpAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
-            exec:
+{{ dict "envAll" $envAll "component" "dhcp_agent" "container" "dhcp_agent" "type" "liveness" "probeTemplate" (include "dhcpAgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
-              command:
-                - python
-                - /tmp/health-probe.py
-                - --config-file
-                - /etc/neutron/neutron.conf
-                - --config-file
-                - /etc/neutron/dhcp_agent.ini
-                - --agent-queue-name
-                - dhcp_agent
-            initialDelaySeconds: 30
-            periodSeconds: 15
-            timeoutSeconds: 65
-          livenessProbe:
-            exec:
-              command:
-                - python
-                - /tmp/health-probe.py
-                - --config-file
-                - /etc/neutron/neutron.conf
-                - --config-file
-                - /etc/neutron/dhcp_agent.ini
-                - --agent-queue-name
-                - dhcp_agent
-                - --liveness-probe
-            initialDelaySeconds: 120
-            periodSeconds: 90
-            timeoutSeconds: 70
           command:
             - /tmp/neutron-dhcp-agent.sh
           volumeMounts:

neutron/templates/daemonset-l3-agent.yaml

@@ -14,6 +14,32 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- define "l3AgentReadinessProbeTemplate" }}
+exec:
+  command:
+    - python
+    - /tmp/health-probe.py
+    - --config-file
+    - /etc/neutron/neutron.conf
+    - --config-file
+    - /etc/neutron/l3_agent.ini
+    - --agent-queue-name
+    - l3_agent
+{{- end }}
+{{- define "l3AgentLivenessProbeTemplate" }}
+exec:
+  command:
+    - python
+    - /tmp/health-probe.py
+    - --config-file
+    - /etc/neutron/neutron.conf
+    - --config-file
+    - /etc/neutron/l3_agent.ini
+    - --agent-queue-name
+    - l3_agent
+    - --liveness-probe
+{{- end }}
+
 {{- define "neutron.l3_agent.daemonset" }}
 {{- $daemonset := index . 0 }}
 {{- $configMapName := index . 1 }}
@@ -65,35 +91,8 @@ spec:
 {{ tuple $envAll "neutron_l3" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.l3 | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "neutron" "container" "neutron_l3_agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
-          readinessProbe:
+{{ dict "envAll" $envAll "component" "l3_agent" "container" "l3_agent" "type" "readiness" "probeTemplate" (include "l3AgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
-            exec:
+{{ dict "envAll" $envAll "component" "l3_agent" "container" "l3_agent" "type" "liveness" "probeTemplate" (include "l3AgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
-              command:
-                - python
-                - /tmp/health-probe.py
-                - --config-file
-                - /etc/neutron/neutron.conf
-                - --config-file
-                - /etc/neutron/l3_agent.ini
-                - --agent-queue-name
-                - l3_agent
-            initialDelaySeconds: 30
-            periodSeconds: 15
-            timeoutSeconds: 65
-          livenessProbe:
-            exec:
-              command:
-                - python
-                - /tmp/health-probe.py
-                - --config-file
-                - /etc/neutron/neutron.conf
-                - --config-file
-                - /etc/neutron/l3_agent.ini
-                - --agent-queue-name
-                - l3_agent
-                - --liveness-probe
-            initialDelaySeconds: 120
-            periodSeconds: 90
-            timeoutSeconds: 70
           command:
             - /tmp/neutron-l3-agent.sh
           volumeMounts:

neutron/templates/daemonset-lb-agent.yaml

@@ -14,6 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- define "lbAgentReadinessProbeTemplate" }}
+exec:
+  command:
+    - bash
+    - -c
+    - 'brctl show'
+{{- end }}
+
 {{- define "neutron.lb_agent.daemonset" }}
 {{- $daemonset := index . 0 }}
 {{- $configMapName := index . 1 }}
@@ -133,14 +141,9 @@ spec:
 {{ tuple $envAll "neutron_linuxbridge_agent" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.lb | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "neutron" "container" "neutron_lb_agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" $envAll "component" "lb_agent" "container" "lb_agent" "type" "readiness" "probeTemplate" (include "lbAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           command:
             - /tmp/neutron-linuxbridge-agent.sh
-          readinessProbe:
-            exec:
-              command:
-                - bash
-                - -c
-                - 'brctl show'
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp

neutron/templates/daemonset-metadata-agent.yaml

@@ -14,6 +14,28 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- define "metadataAgentReadinessProbeTemplate" }}
+exec:
+  command:
+    - python
+    - /tmp/health-probe.py
+    - --config-file
+    - /etc/neutron/neutron.conf
+    - --config-file
+    - /etc/neutron/metadata_agent.ini
+{{- end }}
+{{- define "metadataAgentLivenessProbeTemplate" }}
+exec:
+  command:
+    - python
+    - /tmp/health-probe.py
+    - --config-file
+    - /etc/neutron/neutron.conf
+    - --config-file
+    - /etc/neutron/metadata_agent.ini
+    - --liveness-probe
+{{- end }}
+
 {{- define "neutron.metadata_agent.daemonset" }}
 {{- $daemonset := index . 0 }}
 {{- $configMapName := index . 1 }}
@@ -87,33 +109,10 @@ spec:
         - name: neutron-metadata-agent
 {{ tuple $envAll "neutron_metadata" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.metadata | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "component" "metadata_agent" "container" "metadata_agent" "type" "readiness" "probeTemplate" (include "metadataAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
+{{ dict "envAll" $envAll "component" "metadata_agent" "container" "metadata_agent" "type" "liveness" "probeTemplate" (include "metadataAgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           securityContext:
             privileged: true
-          readinessProbe:
-            exec:
-              command:
-                - python
-                - /tmp/health-probe.py
-                - --config-file
-                - /etc/neutron/neutron.conf
-                - --config-file
-                - /etc/neutron/metadata_agent.ini
-            initialDelaySeconds: 30
-            periodSeconds: 15
-            timeoutSeconds: 35
-          livenessProbe:
-            exec:
-              command:
-                - python
-                - /tmp/health-probe.py
-                - --config-file
-                - /etc/neutron/neutron.conf
-                - --config-file
-                - /etc/neutron/metadata_agent.ini
-                - --liveness-probe
-            initialDelaySeconds: 90
-            periodSeconds: 60
-            timeoutSeconds: 45
           command:
             - /tmp/neutron-metadata-agent.sh
           volumeMounts:

neutron/templates/daemonset-ovs-agent.yaml

@@ -14,6 +14,29 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- define "ovsAgentReadinessProbeTemplate" }}
+# ensures this container can can see a br-int
+# bridge before its marked as ready
+exec:
+  command:
+    - bash
+    - -c
+    - 'ovs-vsctl list-br | grep -q br-int'
+{{- end }}
+{{- define "ovsAgentLivenessProbeTemplate" }}
+exec:
+  command:
+    - python
+    - /tmp/health-probe.py
+    - --config-file
+    - /etc/neutron/neutron.conf
+    - --config-file
+    - /etc/neutron/plugins/ml2/openvswitch_agent.ini
+    - --agent-queue-name
+    - q-agent-notifier-tunnel-update
+    - --liveness-probe
+{{- end }}
+
 {{- define "neutron.ovs_agent.daemonset" }}
 {{- $daemonset := index . 0 }}
 {{- $configMapName := index . 1 }}
@@ -156,33 +179,12 @@ spec:
         - name: neutron-ovs-agent
 {{ tuple $envAll "neutron_openvswitch_agent" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.ovs | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "component" "ovs_agent" "container" "ovs_agent" "type" "readiness" "probeTemplate" (include "ovsAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
+{{ dict "envAll" $envAll "component" "ovs_agent" "container" "ovs_agent" "type" "liveness" "probeTemplate" (include "ovsAgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           securityContext:
             privileged: true
           command:
             - /tmp/neutron-openvswitch-agent.sh
-          # ensures this container can can see a br-int
-          # bridge before its marked as ready
-          readinessProbe:
-            exec:
-              command:
-                - bash
-                - -c
-                - 'ovs-vsctl list-br | grep -q br-int'
-          livenessProbe:
-            exec:
-              command:
-                - python
-                - /tmp/health-probe.py
-                - --config-file
-                - /etc/neutron/neutron.conf
-                - --config-file
-                - /etc/neutron/plugins/ml2/openvswitch_agent.ini
-                - --agent-queue-name
-                - q-agent-notifier-tunnel-update
-                - --liveness-probe
-            initialDelaySeconds: 120
-            periodSeconds: 90
-            timeoutSeconds: 70
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp

neutron/templates/daemonset-sriov-agent.yaml

@@ -14,6 +14,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- define "sriovAgentReadinessProbeTemplate" }}
+exec:
+  command:
+    - python
+    - /tmp/health-probe.py
+    - --config-file
+    - /etc/neutron/neutron.conf
+    - --config-file
+    - /etc/neutron/sriov_agent.ini
+{{- end }}
+
 {{- define "neutron.sriov_agent.daemonset" }}
 {{- $daemonset := index . 0 }}
 {{- $configMapName := index . 1 }}
@@ -64,6 +75,7 @@ spec:
 {{ tuple $envAll "neutron_sriov_agent_init" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.sriov | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "neutron" "container" "neutron_sriov_agent_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" $envAll "component" "sriov_agent" "container" "sriov_agent" "type" "readiness" "probeTemplate" (include "sriovAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           command:
             - /tmp/neutron-sriov-agent-init.sh
           volumeMounts:

neutron/templates/deployment-server.yaml

@@ -14,6 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- define "serverReadinessProbeTemplate" }}
+tcpSocket:
+  port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- end }}
+{{- define "serverLivenessProbeTemplate" }}
+tcpSocket:
+  port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- end }}
+
 {{- if .Values.manifests.deployment_server }}
 {{- $envAll := . }}
 
@@ -63,6 +72,8 @@ spec:
 {{ tuple $envAll "neutron_server" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "neutron" "container" "neutron_server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" $envAll "component" "server" "container" "server" "type" "readiness" "probeTemplate" (include "serverReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
+{{ dict "envAll" $envAll "component" "server" "container" "server" "type" "liveness" "probeTemplate" (include "serverLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           command:
             - /tmp/neutron-server.sh
             - start
@@ -75,13 +86,6 @@ spec:
           ports:
             - name: q-api
               containerPort: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-          readinessProbe:
-            tcpSocket:
-              port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-          livenessProbe:
-            tcpSocket:
-              port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-            initialDelaySeconds: 60
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp

neutron/values.yaml

@@ -301,6 +301,81 @@ dependencies:
           service: local_image_registry
 
 pod:
+  probes:
+    dhcp_agent:
+      dhcp_agent:
+        readiness:
+          enabled: true
+          params:
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            timeoutSeconds: 65
+        liveness:
+          enabled: true
+          params:
+            initialDelaySeconds: 120
+            periodSeconds: 90
+            timeoutSeconds: 70
+    l3_agent:
+      l3_agent:
+        readiness:
+          enabled: true
+          params:
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            timeoutSeconds: 65
+        liveness:
+          enabled: true
+          params:
+            initialDelaySeconds: 120
+            periodSeconds: 90
+            timeoutSeconds: 70
+    lb_agent:
+      lb_agent:
+        readiness:
+          enabled: true
+    metadata_agent:
+      metadata_agent:
+        readiness:
+          enabled: true
+          params:
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            timeoutSeconds: 65
+        liveness:
+          enabled: true
+          params:
+            initialDelaySeconds: 120
+            periodSeconds: 90
+            timeoutSeconds: 70
+    ovs_agent:
+      ovs_agent:
+        readiness:
+          enabled: true
+          params:
+        liveness:
+          enabled: true
+          params:
+            initialDelaySeconds: 120
+            periodSeconds: 90
+            timeoutSeconds: 70
+    sriov_agent:
+      sriov_agent:
+        readiness:
+          enabled: true
+          params:
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            timeoutSeconds: 65
+    server:
+      server:
+        readiness:
+          enabled: true
+          params:
+        liveness:
+          enabled: true
+          params:
+            initialDelaySeconds: 60
   user:
     neutron:
       uid: 42424