From 9a28908d54be76c8e0d35683a63ec5e8ac9fc426 Mon Sep 17 00:00:00 2001 From: Marek Skrobacki Date: Thu, 20 Nov 2025 10:34:55 +0000 Subject: [PATCH] uwsgi: use uWSGI stats for liveness probes in deployments Currently, the OSH uses main uWSGI app to serve responses to the Kubernetes readiness and liveness probes. Unfortunately, this is not sustainable during load. When all of the uWSGI workers are occupied with work for longer than the probe timeout, the liveness probe fails as the request is queued up for too long. This change proposes alternative solution of running the liveness probes against an uWSGI stats endpoint which is a lightweight endpoint served by the master process and is not affected by the workers being busy. It enables the uWSGI stats server on port 1717 in each of the relevant pods and updates the deployments to use the port exposed by those endpoints. This change allows the deployment to use a liveness port that is different from the one dynamically looked up in service catalog. Readiness probes will remain unchanged as it makes sense to check actual application on start. Change-Id: Ie466aafeb4edef72ae1591d91a0f1583636a757c Signed-off-by: Marek Skrobacki --- barbican/values.yaml | 2 ++ cinder/values.yaml | 2 ++ cloudkitty/values.yaml | 2 ++ designate/values.yaml | 2 ++ glance/values.yaml | 2 ++ heat/templates/deployment-api.yaml | 4 ++++ heat/templates/deployment-cfn.yaml | 4 ++++ heat/values.yaml | 13 +++++++++++++ magnum/values.yaml | 2 ++ manila/values.yaml | 2 ++ neutron/templates/deployment-server.yaml | 4 ++++ neutron/values.yaml | 3 +++ nova/values.yaml | 4 ++++ octavia/values.yaml | 2 ++ ovn/values.yaml | 2 ++ placement/templates/deployment.yaml | 4 ++++ placement/values.yaml | 7 +++++++ 17 files changed, 61 insertions(+) diff --git a/barbican/values.yaml b/barbican/values.yaml index c4a6d99a7e..648dbb3a26 100644 --- a/barbican/values.yaml +++ b/barbican/values.yaml @@ -392,6 +392,8 @@ conf: worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/barbican-wsgi-api processes: 1 + stats: 0.0.0.0:1717 + stats-http: true barbican: DEFAULT: transport_url: null diff --git a/cinder/values.yaml b/cinder/values.yaml index e5635d4aff..ee34144d79 100644 --- a/cinder/values.yaml +++ b/cinder/values.yaml @@ -1074,6 +1074,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/cinder-wsgi + stats: 0.0.0.0:1717 + stats-http: true db_purge: before: 30 diff --git a/cloudkitty/values.yaml b/cloudkitty/values.yaml index 14c8b11059..135d685705 100644 --- a/cloudkitty/values.yaml +++ b/cloudkitty/values.yaml @@ -555,6 +555,8 @@ conf: worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/cloudkitty-api processes: 1 + stats: 0.0.0.0:1717 + stats-http: true cloudkitty: DEFAULT: log_config_append: /etc/cloudkitty/logging.conf diff --git a/designate/values.yaml b/designate/values.yaml index db7285f440..53d961f268 100644 --- a/designate/values.yaml +++ b/designate/values.yaml @@ -560,6 +560,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/designate-api-wsgi + stats: 0.0.0.0:1717 + stats-http: true # Names of secrets used by bootstrap and environmental checks secrets: diff --git a/glance/values.yaml b/glance/values.yaml index 82bb9f99b8..ba49f67e71 100644 --- a/glance/values.yaml +++ b/glance/values.yaml @@ -430,6 +430,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/glance-wsgi-api + stats: 0.0.0.0:1717 + stats-http: true network: api: diff --git a/heat/templates/deployment-api.yaml b/heat/templates/deployment-api.yaml index 683ca34af4..296bf667ef 100644 --- a/heat/templates/deployment-api.yaml +++ b/heat/templates/deployment-api.yaml @@ -92,7 +92,11 @@ spec: httpGet: scheme: {{ tuple "orchestration" "service" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }} path: / + {{- if .Values.pod.probes.api.heat_api.liveness.port }} + port: {{ .Values.pod.probes.api.heat_api.liveness.port }} + {{- else }} port: {{ tuple "orchestration" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{- end }} initialDelaySeconds: 10 volumeMounts: - name: pod-tmp diff --git a/heat/templates/deployment-cfn.yaml b/heat/templates/deployment-cfn.yaml index 19a100dab8..92941a6179 100644 --- a/heat/templates/deployment-cfn.yaml +++ b/heat/templates/deployment-cfn.yaml @@ -91,7 +91,11 @@ spec: httpGet: scheme: {{ tuple "cloudformation" "service" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }} path: / + {{- if .Values.pod.probes.cfn.heat_cfn.liveness.port }} + port: {{ .Values.pod.probes.cfn.heat_cfn.liveness.port }} + {{- else }} port: {{ tuple "cloudformation" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{- end }} initialDelaySeconds: 10 volumeMounts: - name: pod-tmp diff --git a/heat/values.yaml b/heat/values.yaml index 8a8ba63b04..1faa339f4a 100644 --- a/heat/values.yaml +++ b/heat/values.yaml @@ -487,6 +487,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/heat-wsgi-api + stats: 0.0.0.0:1717 + stats-http: true heat_api_cfn_uwsgi: uwsgi: add-header: "Connection: close" @@ -503,6 +505,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/heat-wsgi-api-cfn + stats: 0.0.0.0:1717 + stats-http: true network: api: @@ -1171,6 +1175,15 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + probes: + api: + heat_api: + liveness: + port: 1717 + cfn: + heat_cfn: + liveness: + port: 1717 network_policy: heat: diff --git a/magnum/values.yaml b/magnum/values.yaml index 71d61f0bac..b2cade2a6c 100644 --- a/magnum/values.yaml +++ b/magnum/values.yaml @@ -184,6 +184,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/magnum-api-wsgi + stats: 0.0.0.0:1717 + stats-http: true network: api: diff --git a/manila/values.yaml b/manila/values.yaml index 86ef9b5813..27b72f9de9 100644 --- a/manila/values.yaml +++ b/manila/values.yaml @@ -866,6 +866,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 module: "manila.wsgi.api:application" + stats: 0.0.0.0:1717 + stats-http: true # Names of secrets used by bootstrap and environmental checks secrets: diff --git a/neutron/templates/deployment-server.yaml b/neutron/templates/deployment-server.yaml index 07de4e8443..4bc8c3e520 100644 --- a/neutron/templates/deployment-server.yaml +++ b/neutron/templates/deployment-server.yaml @@ -39,7 +39,11 @@ initialDelaySeconds: 30 httpGet: scheme: {{ tuple "network" "service" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }} path: / + {{- if .Values.pod.probes.server.server.liveness.port }} + port: {{ .Values.pod.probes.server.server.liveness.port }} + {{ else }} port: {{ tuple "network" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{- end}} {{- end }} {{- end }} diff --git a/neutron/values.yaml b/neutron/values.yaml index fb9ca87874..a53de78ef2 100644 --- a/neutron/values.yaml +++ b/neutron/values.yaml @@ -529,6 +529,7 @@ pod: timeoutSeconds: 10 liveness: enabled: true + port: 1717 params: initialDelaySeconds: 60 periodSeconds: 15 @@ -1387,6 +1388,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/neutron-api + stats: 0.0.0.0:1717 + stats-http: true policy: {} api_audit_map: DEFAULT: diff --git a/nova/values.yaml b/nova/values.yaml index d25ce3219e..6207ae6f66 100644 --- a/nova/values.yaml +++ b/nova/values.yaml @@ -1620,6 +1620,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/nova-api-wsgi + stats: 0.0.0.0:1717 + stats-http: true nova_metadata_uwsgi: uwsgi: add-header: "Connection: close" @@ -1636,6 +1638,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/nova-metadata-wsgi + stats: 0.0.0.0:1717 + stats-http: true # Names of secrets used by bootstrap and environmental checks secrets: diff --git a/octavia/values.yaml b/octavia/values.yaml index c994add9c5..2c43330bac 100644 --- a/octavia/values.yaml +++ b/octavia/values.yaml @@ -401,6 +401,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 module: "octavia.wsgi.api:application" + stats: 0.0.0.0:1717 + stats-http: true secrets: identity: diff --git a/ovn/values.yaml b/ovn/values.yaml index 64e077ff82..4870eeae35 100644 --- a/ovn/values.yaml +++ b/ovn/values.yaml @@ -104,6 +104,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/neutron-ovn-network-logging-parser-wsgi + stats: 0.0.0.0:1717 + stats-http: true vector: | [sources.file_logs] type = "file" diff --git a/placement/templates/deployment.yaml b/placement/templates/deployment.yaml index decf7ddb81..9cf1b07ef3 100644 --- a/placement/templates/deployment.yaml +++ b/placement/templates/deployment.yaml @@ -108,7 +108,11 @@ spec: httpGet: scheme: {{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }} path: / + {{- if .Values.pod.probes.placement.api.liveness.port }} + port: {{ .Values.pod.probes.placement.api.liveness.port }} + {{- else }} port: {{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{- end }} initialDelaySeconds: 5 periodSeconds: 10 volumeMounts: diff --git a/placement/values.yaml b/placement/values.yaml index cae908d129..835c5202e7 100644 --- a/placement/values.yaml +++ b/placement/values.yaml @@ -153,6 +153,8 @@ conf: thunder-lock: true worker-reload-mercy: 80 wsgi-file: /var/lib/openstack/bin/placement-api + stats: 0.0.0.0:1717 + stats-http: true endpoints: cluster_domain_suffix: cluster.local local_image_registry: @@ -374,6 +376,11 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + probes: + placement: + api: + liveness: + port: 1717 secrets: identity: