Browse Source

Control bind9 and nginx resource usage

The named and nginx processes both try to use all available CPUs. In
addition, there is a bug in named that sometimes causes it to spin on a
FUTEX, pegging the CPU.

This change constrains those processes to a single CPU (overridable in
values.yaml), and includes /etc/bind/bind.keys in named.conf to avoid
the CPU spike.

Change-Id: I4a278023f5c0dd5e7bdee46891591b278f2ddcad
changes/93/816593/4
Phil Sphicas 6 months ago
parent
commit
50b3d68905
  1. 4
      charts/maas/templates/bin/_start.sh.tpl
  2. 4
      charts/maas/templates/configmap-etc.yaml
  3. 8
      charts/maas/templates/etc/_bind9.tpl
  4. 37
      charts/maas/templates/etc/_nginx.conf.tpl
  5. 8
      charts/maas/templates/statefulset-rack.yaml
  6. 4
      charts/maas/templates/statefulset-region.yaml
  7. 8
      charts/maas/values.yaml
  8. 3
      images/maas-rack-controller/Dockerfile
  9. 3
      images/maas-region-controller/Dockerfile

4
charts/maas/templates/bin/_start.sh.tpl

@ -21,11 +21,11 @@ env > /tmp/env
# Ensure PVC volumes have correct ownership
# Also restore the subdirectory structure and any default files
# (i.e. /var/lib/maas/http/nginx.conf)
# that are not overridden
chown maas:maas ~maas/
chown maas:maas /etc/maas
[[ -r /opt/maas/var-lib-maas.tgz ]] && tar -C/ -xvzf /opt/maas/var-lib-maas.tgz
[[ -r /opt/maas/var-lib-maas.tgz ]] && tar -C/ -xvzf /opt/maas/var-lib-maas.tgz || true
[[ -d ~maas/boot-resources ]] && chown -R maas:maas ~maas/boot-resources
# MAAS must be able to ssh to libvirt hypervisors

4
charts/maas/templates/configmap-etc.yaml

@ -36,6 +36,10 @@ data:
{{- if .Values.conf.bind.append -}}
{{ .Values.conf.bind.append | indent 4 }}
{{- end }}
bind9: |
{{ tuple "etc/_bind9.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
nginx.conf: |
{{ tuple "etc/_nginx.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
regiond.conf: |
{{ tuple "etc/_regiond.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
nsswitch.conf: |

8
charts/maas/templates/etc/_bind9.tpl

@ -0,0 +1,8 @@
{{/* file location: /etc/default/bind9 */}}
{{- $cpus := index .Values.conf.bind "cpus" -}}
#
# run resolvconf?
RESOLVCONF=no
# startup options for the server
OPTIONS="-u bind {{- if $cpus }} -n {{ $cpus }}{{ end }}"

37
charts/maas/templates/etc/_nginx.conf.tpl

@ -0,0 +1,37 @@
{{/* file location: /var/lib/maas/http/nginx.conf */}}
{{- $worker_processes := index .Values.conf.nginx "worker_processes" | default "auto" -}}
{{- $worker_connections := index .Values.conf.nginx "worker_connections" | default 768 -}}
pid /run/maas-http.pid;
worker_processes {{ $worker_processes }};
error_log /var/log/maas/http/error.log;
events {
worker_connections {{ $worker_connections }};
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
client_max_body_size 10M;
include /etc/nginx/mime.types;
default_type application/octet-stream;
access_log /var/log/maas/http/access.log;
gzip on;
include /var/lib/maas/http/*.nginx.conf;
# LP: #1796224 and #1869067 - Use different paths otherwise this will
# conflict with the system's nginx daemon.
client_body_temp_path /var/lib/maas/http/body;
fastcgi_temp_path /var/lib/maas/http/fastcgi;
proxy_temp_path /var/lib/maas/http/proxy;
scgi_temp_path /var/lib/maas/http/scgi;
uwsgi_temp_path /var/lib/maas/http/uwsgi;
}

8
charts/maas/templates/statefulset-rack.yaml

@ -138,6 +138,14 @@ spec:
subPath: PRIVATE_KEY
mountPath: /var/lib/maas/id_rsa
{{- end }}
- name: maas-etc
mountPath: /var/lib/maas/http/nginx.conf
subPath: nginx.conf
readOnly: true
- name: maas-etc
mountPath: /etc/default/bind9
subPath: bind9
readOnly: true
{{ if $mounts_maas_rack.volumeMounts }}{{ toYaml $mounts_maas_rack.volumeMounts | indent 12 }}{{ end }}
volumes:
- name: host-sys-fs-cgroup

4
charts/maas/templates/statefulset-region.yaml

@ -169,6 +169,10 @@ spec:
subPath: curtin
readOnly: true
{{- end }}
- name: maas-etc
mountPath: /etc/default/bind9
subPath: bind9
readOnly: true
{{- if $mounts_maas_region.volumeMounts }}{{ toYaml $mounts_maas_region.volumeMounts | indent 12 }}{{ end }}
volumes:
- name: host-sys-fs-cgroup

8
charts/maas/values.yaml

@ -181,6 +181,14 @@ conf:
bind:
override:
append:
# 'cpus: n' number of CPUs for bind to use
# 'cpus: ""' to revert to the default (all of them)
cpus: 1
nginx:
# 'worker_processes: auto' (the maas default) launches one worker per core
worker_processes: 1
# 'worker_connections: 768' is the MAAS default, 512 is the nginx default
worker_connections: 768
curtin:
override: true
late_commands: {}

3
images/maas-rack-controller/Dockerfile

@ -74,5 +74,8 @@ RUN systemctl enable journalctl-to-tty.service
# quiet sudo for the maas user
RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log
# avoid triggering bind9 high cpu utilization bug
RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf
# initalize systemd
CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]

3
images/maas-region-controller/Dockerfile

@ -82,5 +82,8 @@ RUN systemctl enable journalctl-to-tty.service
# quiet sudo for the maas user
RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log
# avoid triggering bind9 high cpu utilization bug
RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf
# initalize systemd
CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]

Loading…
Cancel
Save