de9f8415d7
There are several kubernetes bugs [0,1,2] involving connection problems that seem related to the Go net/http2 library, where the stream state and connection state can get out of sync. This can manifest as a kubelet issue, where the node status gets stuck in a NotReady state, but can also happen elsewhere. In newer versions of the Go libraries some issues are fixed [3,4], but the fixes are not present in k8s 1.18. This change disables http2 in kube-apiserver and webhook-apiserver. This should be sufficient to avoid the majority of the issues, as disabling on one side of the connection is enough, and apiserver is generally either the client or the server. 0: https://github.com/kubernetes/kubernetes/issues/87615 1: https://github.com/kubernetes/kubernetes/issues/80313 2: https://github.com/kubernetes/client-go/issues/374 3: https://github.com/golang/go/issues/40423 4: https://github.com/golang/go/issues/40201 Change-Id: Id693a7201acffccbc4b3db8f4e4b96290fd50288
281 lines
13 KiB
YAML
281 lines
13 KiB
YAML
{{/*
|
|
Copyright 2018 The Openstack-Helm Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/}}
|
|
|
|
{{/*
|
|
These local.* templates may be moved out of this chart into helm-toolkit
|
|
in the future if there is desire to generalize this pattern. Otherwise
|
|
in the future they will be moved into a separate helpers file.
|
|
*/}}
|
|
|
|
{{- define "local.tls_volume_name" -}}
|
|
{{- $group := index . 0 -}}
|
|
{{- $type := index . 1 -}}
|
|
tls-{{ $group | replace "_" "-" }}-{{ $type | replace "_" "-" }}
|
|
{{- end -}}
|
|
|
|
{{- define "local.attach_all_bundles" }}
|
|
{{- $envAll := . }}
|
|
{{- range $group, $certs := $envAll.Values.certificates }}
|
|
{{- range $type, $bundle := . }}
|
|
{{ tuple $group $type $envAll | include "local.attach_cert_bundle" }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
|
|
{{- define "local.attach_cert_bundle" }}
|
|
{{- $group := index . 0 }}
|
|
{{- $type := index . 1 }}
|
|
{{- $envAll := index . 2 }}
|
|
- name: {{ tuple $group $type | include "local.tls_volume_name" }}
|
|
secret:
|
|
secretName: {{ tuple $group $type $envAll | include "local.tls_secret_name" }}
|
|
defaultMode: 0444
|
|
{{ end }}
|
|
|
|
{{- define "local.mount_all_bundles" }}
|
|
{{- $basepath := index . 0 }}
|
|
{{- $envAll := index . 1 }}
|
|
{{- range $group, $certs := $envAll.Values.certificates }}
|
|
{{- range $type, $bundle := . }}
|
|
{{ tuple $group $type $basepath $envAll | include "local.mount_cert_bundle" }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
|
|
{{- define "local.mount_cert_bundle" }}
|
|
{{- $group := index . 0 }}
|
|
{{- $type := index . 1 }}
|
|
{{- $basepath := index . 2 }}
|
|
{{- $envAll := index . 3 }}
|
|
{{- $bundle := index $envAll.Values "certificates" $group $type }}
|
|
{{- range tuple "ca" "cert" "key" }}
|
|
{{- if hasKey $bundle . }}
|
|
{{ tuple $group $type . $basepath $envAll | include "local.mount_cert_file" }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
|
|
{{- define "local.mount_cert_file" }}
|
|
{{- $group := index . 0 }}
|
|
{{- $type := index . 1 }}
|
|
{{- $member := index . 2 }}
|
|
{{- $basepath := index . 3 }}
|
|
{{- $envAll := index . 4 }}
|
|
- name: {{ tuple $group $type | include "local.tls_volume_name" }}
|
|
mountPath: {{ tuple $group $type $basepath $member $envAll | include "local.cert_bundle_path" }}
|
|
{{- if eq $member "ca" }}
|
|
subPath: ca.crt
|
|
{{- else if eq $member "cert" }}
|
|
subPath: tls.crt
|
|
{{- else if eq $member "key" }}
|
|
subPath: tls.key
|
|
{{- end }}
|
|
readOnly: true
|
|
{{- end }}
|
|
|
|
{{- define "local.cert_bundle_path" -}}
|
|
{{- $group := index . 0 -}}
|
|
{{- $type := index . 1 -}}
|
|
{{- $basepath := index . 2 -}}
|
|
{{- $member := index . 3 -}}
|
|
{{- $envAll := index . 4 -}}
|
|
{{ $basepath }}/{{ $group }}-{{ $type }}-{{ $member }}.pem
|
|
{{- end -}}
|
|
|
|
{{- if .Values.manifests.deployment }}
|
|
{{- $envAll := . }}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: {{ .Release.Name }}-apiserver-webhook
|
|
labels:
|
|
annotations:
|
|
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 4 }}
|
|
{{ tuple $envAll "kubernetes-keystone-webhook" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
|
|
spec:
|
|
replicas: {{ $envAll.Values.pod.replicas.api }}
|
|
selector:
|
|
matchLabels:
|
|
{{ tuple $envAll "kubernetes-keystone-webhook" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
|
|
template:
|
|
metadata:
|
|
labels:
|
|
{{ tuple $envAll "kubernetes-keystone-webhook" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
|
|
annotations:
|
|
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
|
|
{{ dict "envAll" $envAll "podName" "apiserver-webhook" "containerNames" (list "apiserver" "webhook") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
|
|
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
|
|
configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
|
|
spec:
|
|
nodeSelector:
|
|
{{ .Values.labels.kubernetes_apiserver.node_selector_key }}: {{ .Values.labels.kubernetes_apiserver.node_selector_value }}
|
|
affinity:
|
|
{{ tuple $envAll "kubernetes-keystone-webhook" "api" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
|
|
dnsPolicy: ClusterFirst
|
|
{{ dict "envAll" $envAll "application" "apiserver_webhook" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
|
|
containers:
|
|
- name: apiserver
|
|
image: {{ .Values.images.tags.apiserver }}
|
|
{{ tuple $envAll $envAll.Values.pod.resources.kubernetes_apiserver | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
|
|
{{ dict "envAll" $envAll "application" "apiserver_webhook" "container" "apiserver" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
|
|
env:
|
|
- name: POD_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.podIP
|
|
- name: NODENAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.apiserver | indent 12 }}
|
|
command:
|
|
{{- range .Values.command_prefix }}
|
|
- {{ . }}
|
|
{{- end }}
|
|
{{- if $envAll.Values.apiserver_webhook.logging.log_level }}
|
|
- --v={{ $envAll.Values.apiserver_webhook.logging.log_level }}
|
|
{{- end }}
|
|
- --service-cluster-ip-range={{ $envAll.Values.network.service_cidr }}
|
|
- --authorization-mode=Webhook
|
|
- --advertise-address=$(POD_IP)
|
|
- --anonymous-auth=false
|
|
- --endpoint-reconciler-type=none
|
|
- --bind-address=$(POD_IP)
|
|
- --secure-port={{ tuple "webhook_apiserver" "podport" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
|
|
- --insecure-port=0
|
|
- --tls-cert-file={{ tuple "apiserver_webhook_pod" "server" $envAll.Values.conf.paths.pki "cert" $envAll | include "local.cert_bundle_path" }}
|
|
- --tls-private-key-file={{ tuple "apiserver_webhook_pod" "server" $envAll.Values.conf.paths.pki "key" $envAll | include "local.cert_bundle_path" }}
|
|
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
|
|
- --kubelet-certificate-authority={{ tuple "kubelet" "server" $envAll.Values.conf.paths.pki "ca" $envAll | include "local.cert_bundle_path" }}
|
|
- --kubelet-client-certificate={{ tuple "kubelet" "client" $envAll.Values.conf.paths.pki "cert" $envAll | include "local.cert_bundle_path" }}
|
|
- --kubelet-client-key={{ tuple "kubelet" "client" $envAll.Values.conf.paths.pki "key" $envAll | include "local.cert_bundle_path" }}
|
|
- --etcd-servers={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
|
|
- --etcd-cafile={{ tuple "etcd" "server" $envAll.Values.conf.paths.pki "ca" $envAll | include "local.cert_bundle_path" }}
|
|
- --etcd-certfile={{ tuple "etcd" "client" $envAll.Values.conf.paths.pki "cert" $envAll | include "local.cert_bundle_path" }}
|
|
- --etcd-keyfile={{ tuple "etcd" "client" $envAll.Values.conf.paths.pki "key" $envAll | include "local.cert_bundle_path" }}
|
|
- --allow-privileged=true
|
|
- --service-account-key-file={{ $envAll.Values.conf.paths.sapubkey }}
|
|
- --authentication-token-webhook-config-file={{ $envAll.Values.conf.paths.conf }}
|
|
- --authorization-webhook-config-file={{ $envAll.Values.conf.paths.conf }}
|
|
{{- range $key, $val := .Values.conf.apiserver }}
|
|
{{- if hasKey $val "command_options" }}
|
|
{{- range $val.command_options }}
|
|
- {{ . }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
readinessProbe:
|
|
tcpSocket:
|
|
port: {{ tuple "webhook_apiserver" "podport" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
|
|
{{ $envAll.Values.pod.probes.readinessProbe | toYaml | indent 12 }}
|
|
livenessProbe:
|
|
tcpSocket:
|
|
port: {{ tuple "webhook_apiserver" "podport" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
|
|
{{ $envAll.Values.pod.probes.livenessProbe | toYaml | indent 12 }}
|
|
volumeMounts:
|
|
- name: etc-apiserver
|
|
mountPath: {{ $envAll.Values.conf.paths.base }}
|
|
- name: apiserver-dynamic-config
|
|
mountPath: {{ $envAll.Values.conf.paths.base }}dynamic
|
|
readOnly: true
|
|
- name: etc-apiserver-pki
|
|
mountPath: {{ $envAll.Values.conf.paths.pki }}
|
|
- name: configmap-etc
|
|
mountPath: {{ $envAll.Values.conf.paths.sapubkey }}
|
|
subPath: service-account.pub
|
|
readOnly: true
|
|
- name: configmap-etc
|
|
mountPath: {{ $envAll.Values.conf.paths.conf }}
|
|
subPath: webhook.kubeconfig
|
|
readOnly: true
|
|
- name: configmap-etc
|
|
mountPath: {{ $envAll.Values.conf.paths.encryption_provider }}
|
|
subPath: encryption_provider.json
|
|
readOnly: true
|
|
{{ tuple "keystone_webhook" "server" "ca" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_file" | indent 12 }}
|
|
{{ tuple "apiserver_webhook_pod" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
|
|
{{ tuple "kubelet" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
|
|
{{ tuple "kubelet" "client" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
|
|
{{ tuple "etcd" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
|
|
{{ tuple "etcd" "client" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
|
|
- name: webhook
|
|
{{ tuple $envAll "kubernetes_keystone_webhook" | include "helm-toolkit.snippets.image" | indent 10 }}
|
|
{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
|
|
{{ dict "envAll" $envAll "application" "apiserver_webhook" "container" "webhook" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
|
|
command:
|
|
- /tmp/webhook_start.sh
|
|
env:
|
|
{{- with $env := dict "ksUserSecret" .Values.secrets.identity.webhook }}
|
|
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
|
|
{{- end }}
|
|
- name: SERVER_CERT_FILE
|
|
value: {{ tuple "keystone_webhook" "server" $envAll.Values.conf.paths.pki "cert" $envAll | include "local.cert_bundle_path" | quote }}
|
|
- name: SERVER_KEY_FILE
|
|
value: {{ tuple "keystone_webhook" "server" $envAll.Values.conf.paths.pki "key" $envAll | include "local.cert_bundle_path" | quote }}
|
|
- name: POLICY_FILE
|
|
value: {{ $envAll.Values.conf.paths.policy | quote }}
|
|
- name: SERVER_PORT
|
|
value: {{ tuple "webhook_apiserver" "podport" "webhook" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
|
|
{{- if hasKey .Values.certificates "keystone" }}
|
|
- name: KEYSTONE_CA_FILE
|
|
value: {{ tuple "keystone" "server" $envAll.Values.conf.paths.pki "ca" $envAll | include "local.cert_bundle_path" | quote }}
|
|
{{- end }}
|
|
volumeMounts:
|
|
- name: etc-webhook
|
|
mountPath: {{ $envAll.Values.conf.paths.base }}
|
|
- name: etc-webhook-pki
|
|
mountPath: {{ $envAll.Values.conf.paths.pki }}
|
|
- name: configmap-etc
|
|
mountPath: {{ $envAll.Values.conf.paths.policy }}
|
|
subPath: policy.json
|
|
readOnly: true
|
|
- name: configmap-bin
|
|
mountPath: /tmp/webhook_start.sh
|
|
subPath: webhook_start.sh
|
|
readOnly: true
|
|
{{ tuple "keystone_webhook" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
|
|
volumes:
|
|
{{- if hasKey .Values.certificates "keystone" }}
|
|
{{ tuple "keystone" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
|
|
{{- end }}
|
|
{{ include "local.attach_all_bundles" $envAll | indent 8 }}
|
|
- name: etc-apiserver
|
|
emptyDir: {}
|
|
- name: etc-apiserver-pki
|
|
emptyDir: {}
|
|
- name: apiserver-dynamic-config
|
|
configMap:
|
|
name: {{ .Release.Name }}-dynamic-config
|
|
defaultMode: 0444
|
|
- name: etc-webhook
|
|
emptyDir: {}
|
|
- name: etc-webhook-pki
|
|
emptyDir: {}
|
|
- name: configmap-etc
|
|
configMap:
|
|
name: {{ .Release.Name }}-etc
|
|
defaultMode: 0444
|
|
- name: configmap-bin
|
|
configMap:
|
|
name: {{ .Release.Name }}-bin
|
|
defaultMode: 0555
|
|
- name: tls-apiserver-webhook-public-server
|
|
secret:
|
|
defaultMode: 292
|
|
secretName: {{ .Values.secrets.tls.webhook_apiserver.api.public }}
|
|
{{- end }}
|