Control bind9 and nginx resource usage

The named and nginx processes both try to use all available CPUs. In
addition, there is a bug in named that sometimes causes it to spin on a
FUTEX, pegging the CPU.

This change constrains those processes to a single CPU (overridable in
values.yaml), and includes /etc/bind/bind.keys in named.conf to avoid
the CPU spike.

Change-Id: I4a278023f5c0dd5e7bdee46891591b278f2ddcad
This commit is contained in:
Phil Sphicas 2021-11-03 17:02:32 -07:00
parent 666567eae5
commit 50b3d68905
9 changed files with 77 additions and 2 deletions

View File

@ -21,11 +21,11 @@ env > /tmp/env
# Ensure PVC volumes have correct ownership # Ensure PVC volumes have correct ownership
# Also restore the subdirectory structure and any default files # Also restore the subdirectory structure and any default files
# (i.e. /var/lib/maas/http/nginx.conf) # that are not overridden
chown maas:maas ~maas/ chown maas:maas ~maas/
chown maas:maas /etc/maas chown maas:maas /etc/maas
[[ -r /opt/maas/var-lib-maas.tgz ]] && tar -C/ -xvzf /opt/maas/var-lib-maas.tgz [[ -r /opt/maas/var-lib-maas.tgz ]] && tar -C/ -xvzf /opt/maas/var-lib-maas.tgz || true
[[ -d ~maas/boot-resources ]] && chown -R maas:maas ~maas/boot-resources [[ -d ~maas/boot-resources ]] && chown -R maas:maas ~maas/boot-resources
# MAAS must be able to ssh to libvirt hypervisors # MAAS must be able to ssh to libvirt hypervisors

View File

@ -36,6 +36,10 @@ data:
{{- if .Values.conf.bind.append -}} {{- if .Values.conf.bind.append -}}
{{ .Values.conf.bind.append | indent 4 }} {{ .Values.conf.bind.append | indent 4 }}
{{- end }} {{- end }}
bind9: |
{{ tuple "etc/_bind9.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
nginx.conf: |
{{ tuple "etc/_nginx.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
regiond.conf: | regiond.conf: |
{{ tuple "etc/_regiond.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{ tuple "etc/_regiond.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
nsswitch.conf: | nsswitch.conf: |

View File

@ -0,0 +1,8 @@
{{/* file location: /etc/default/bind9 */}}
{{- $cpus := index .Values.conf.bind "cpus" -}}
#
# run resolvconf?
RESOLVCONF=no
# startup options for the server
OPTIONS="-u bind {{- if $cpus }} -n {{ $cpus }}{{ end }}"

View File

@ -0,0 +1,37 @@
{{/* file location: /var/lib/maas/http/nginx.conf */}}
{{- $worker_processes := index .Values.conf.nginx "worker_processes" | default "auto" -}}
{{- $worker_connections := index .Values.conf.nginx "worker_connections" | default 768 -}}
pid /run/maas-http.pid;
worker_processes {{ $worker_processes }};
error_log /var/log/maas/http/error.log;
events {
worker_connections {{ $worker_connections }};
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
client_max_body_size 10M;
include /etc/nginx/mime.types;
default_type application/octet-stream;
access_log /var/log/maas/http/access.log;
gzip on;
include /var/lib/maas/http/*.nginx.conf;
# LP: #1796224 and #1869067 - Use different paths otherwise this will
# conflict with the system's nginx daemon.
client_body_temp_path /var/lib/maas/http/body;
fastcgi_temp_path /var/lib/maas/http/fastcgi;
proxy_temp_path /var/lib/maas/http/proxy;
scgi_temp_path /var/lib/maas/http/scgi;
uwsgi_temp_path /var/lib/maas/http/uwsgi;
}

View File

@ -138,6 +138,14 @@ spec:
subPath: PRIVATE_KEY subPath: PRIVATE_KEY
mountPath: /var/lib/maas/id_rsa mountPath: /var/lib/maas/id_rsa
{{- end }} {{- end }}
- name: maas-etc
mountPath: /var/lib/maas/http/nginx.conf
subPath: nginx.conf
readOnly: true
- name: maas-etc
mountPath: /etc/default/bind9
subPath: bind9
readOnly: true
{{ if $mounts_maas_rack.volumeMounts }}{{ toYaml $mounts_maas_rack.volumeMounts | indent 12 }}{{ end }} {{ if $mounts_maas_rack.volumeMounts }}{{ toYaml $mounts_maas_rack.volumeMounts | indent 12 }}{{ end }}
volumes: volumes:
- name: host-sys-fs-cgroup - name: host-sys-fs-cgroup

View File

@ -169,6 +169,10 @@ spec:
subPath: curtin subPath: curtin
readOnly: true readOnly: true
{{- end }} {{- end }}
- name: maas-etc
mountPath: /etc/default/bind9
subPath: bind9
readOnly: true
{{- if $mounts_maas_region.volumeMounts }}{{ toYaml $mounts_maas_region.volumeMounts | indent 12 }}{{ end }} {{- if $mounts_maas_region.volumeMounts }}{{ toYaml $mounts_maas_region.volumeMounts | indent 12 }}{{ end }}
volumes: volumes:
- name: host-sys-fs-cgroup - name: host-sys-fs-cgroup

View File

@ -181,6 +181,14 @@ conf:
bind: bind:
override: override:
append: append:
# 'cpus: n' number of CPUs for bind to use
# 'cpus: ""' to revert to the default (all of them)
cpus: 1
nginx:
# 'worker_processes: auto' (the maas default) launches one worker per core
worker_processes: 1
# 'worker_connections: 768' is the MAAS default, 512 is the nginx default
worker_connections: 768
curtin: curtin:
override: true override: true
late_commands: {} late_commands: {}

View File

@ -74,5 +74,8 @@ RUN systemctl enable journalctl-to-tty.service
# quiet sudo for the maas user # quiet sudo for the maas user
RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log
# avoid triggering bind9 high cpu utilization bug
RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf
# initalize systemd # initalize systemd
CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"] CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]

View File

@ -82,5 +82,8 @@ RUN systemctl enable journalctl-to-tty.service
# quiet sudo for the maas user # quiet sudo for the maas user
RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log
# avoid triggering bind9 high cpu utilization bug
RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf
# initalize systemd # initalize systemd
CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"] CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]