openstack/openstack-helm
Code Issues Proposed changes

uwsgi: use uWSGI stats for liveness probes in deployments

Browse Source 
Currently, the OSH uses main uWSGI app to serve responses to the
Kubernetes readiness and liveness probes. Unfortunately, this is not
sustainable during load. When all of the uWSGI workers are occupied with
work for longer than the probe timeout, the liveness probe fails as the
request is queued up for too long.

This change proposes alternative solution of running the liveness probes
against an uWSGI stats endpoint which is a lightweight endpoint served
by the master process and is not affected by the workers being busy.

It enables the uWSGI stats server on port 1717 in each of the
relevant pods and updates the deployments to use the port exposed by
those endpoints.

This change allows the deployment to use a liveness port that is
different from the one dynamically looked up in service catalog.

Readiness probes will remain unchanged as it makes sense to check actual
application on start.

Change-Id: Ie466aafeb4edef72ae1591d91a0f1583636a757c
Signed-off-by: Marek Skrobacki <marek.skrobacki@rackspace.co.uk>
This commit is contained in:
Marek Skrobacki
2025-11-20 10:34:55 +00:00
parent a1599e7175
commit 9a28908d54
17 changed files with 61 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
barbican/values.yaml
View File

@@ -392,6 +392,8 @@ conf:
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/barbican-wsgi-api
processes: 1
stats: 0.0.0.0:1717
stats-http: true
barbican:
DEFAULT:
transport_url: null

2
cinder/values.yaml
View File

@@ -1074,6 +1074,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/cinder-wsgi
stats: 0.0.0.0:1717
stats-http: true
db_purge:
before: 30

2
cloudkitty/values.yaml
View File

@@ -555,6 +555,8 @@ conf:
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/cloudkitty-api
processes: 1
stats: 0.0.0.0:1717
stats-http: true
cloudkitty:
DEFAULT:
log_config_append: /etc/cloudkitty/logging.conf

2
designate/values.yaml
View File

@@ -560,6 +560,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/designate-api-wsgi
stats: 0.0.0.0:1717
stats-http: true
# Names of secrets used by bootstrap and environmental checks
secrets:

2
glance/values.yaml
View File

@@ -430,6 +430,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/glance-wsgi-api
stats: 0.0.0.0:1717
stats-http: true
network:
api:

4
heat/templates/deployment-api.yaml
View File

@@ -92,7 +92,11 @@ spec:
httpGet:
scheme: {{ tuple "orchestration" "service" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }}
path: /
{{- if .Values.pod.probes.api.heat_api.liveness.port }}
port: {{ .Values.pod.probes.api.heat_api.liveness.port }}
{{- else }}
port: {{ tuple "orchestration" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- end }}
initialDelaySeconds: 10
volumeMounts:
- name: pod-tmp

4
heat/templates/deployment-cfn.yaml
View File

@@ -91,7 +91,11 @@ spec:
httpGet:
scheme: {{ tuple "cloudformation" "service" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }}
path: /
{{- if .Values.pod.probes.cfn.heat_cfn.liveness.port }}
port: {{ .Values.pod.probes.cfn.heat_cfn.liveness.port }}
{{- else }}
port: {{ tuple "cloudformation" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- end }}
initialDelaySeconds: 10
volumeMounts:
- name: pod-tmp

13
heat/values.yaml
View File

@@ -487,6 +487,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/heat-wsgi-api
stats: 0.0.0.0:1717
stats-http: true
heat_api_cfn_uwsgi:
uwsgi:
add-header: "Connection: close"
@@ -503,6 +505,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/heat-wsgi-api-cfn
stats: 0.0.0.0:1717
stats-http: true
network:
api:
@@ -1171,6 +1175,15 @@ pod:
limits:
memory: "1024Mi"
cpu: "2000m"
probes:
api:
heat_api:
liveness:
port: 1717
cfn:
heat_cfn:
liveness:
port: 1717
network_policy:
heat:

2
magnum/values.yaml
View File

@@ -184,6 +184,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/magnum-api-wsgi
stats: 0.0.0.0:1717
stats-http: true
network:
api:

2
manila/values.yaml
View File

@@ -866,6 +866,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
module: "manila.wsgi.api:application"
stats: 0.0.0.0:1717
stats-http: true
# Names of secrets used by bootstrap and environmental checks
secrets:

4
neutron/templates/deployment-server.yaml
View File

@@ -39,7 +39,11 @@ initialDelaySeconds: 30
httpGet:
scheme: {{ tuple "network" "service" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }}
path: /
{{- if .Values.pod.probes.server.server.liveness.port }}
port: {{ .Values.pod.probes.server.server.liveness.port }}
{{ else }}
port: {{ tuple "network" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- end}}
{{- end }}
{{- end }}

3
neutron/values.yaml
View File

@@ -529,6 +529,7 @@ pod:
timeoutSeconds: 10
liveness:
enabled: true
port: 1717
params:
initialDelaySeconds: 60
periodSeconds: 15
@@ -1387,6 +1388,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/neutron-api
stats: 0.0.0.0:1717
stats-http: true
policy: {}
api_audit_map:
DEFAULT:

4
nova/values.yaml
View File

@@ -1620,6 +1620,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/nova-api-wsgi
stats: 0.0.0.0:1717
stats-http: true
nova_metadata_uwsgi:
uwsgi:
add-header: "Connection: close"
@@ -1636,6 +1638,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/nova-metadata-wsgi
stats: 0.0.0.0:1717
stats-http: true
# Names of secrets used by bootstrap and environmental checks
secrets:

2
octavia/values.yaml
View File

@@ -401,6 +401,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
module: "octavia.wsgi.api:application"
stats: 0.0.0.0:1717
stats-http: true
secrets:
identity:

2
ovn/values.yaml
View File

@@ -104,6 +104,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/neutron-ovn-network-logging-parser-wsgi
stats: 0.0.0.0:1717
stats-http: true
vector: |
[sources.file_logs]
type = "file"

4
placement/templates/deployment.yaml
View File

@@ -108,7 +108,11 @@ spec:
httpGet:
scheme: {{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }}
path: /
{{- if .Values.pod.probes.placement.api.liveness.port }}
port: {{ .Values.pod.probes.placement.api.liveness.port }}
{{- else }}
port: {{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- end }}
initialDelaySeconds: 5
periodSeconds: 10
volumeMounts:

7
placement/values.yaml
View File

@@ -153,6 +153,8 @@ conf:
thunder-lock: true
worker-reload-mercy: 80
wsgi-file: /var/lib/openstack/bin/placement-api
stats: 0.0.0.0:1717
stats-http: true
endpoints:
cluster_domain_suffix: cluster.local
local_image_registry:
@@ -374,6 +376,11 @@ pod:
limits:
memory: "1024Mi"
cpu: "2000m"
probes:
placement:
api:
liveness:
port: 1717
secrets:
identity: