Browse Source

Add EventRateLimit admission controller

Add the EventRateLimit admission controller, to allow operators to
define rate limits for the k8s API server at the server, namespace,
or user account level.

This also
* cleans up some of the parameters passed into the API server
* replaces the deprecated --admission-control parameter
* applies --repair-malformed-updates consistently, incl examples
* removes unused batch/v2alpha1 runtime config
* https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/
* removes duplicate --service-cluster-ip-range setting

This PS adds EventRateLimits to the bootstrap and anchor API
servers; future work will need to add it to the Keystone
Webhook API server.

Change-Id: I32a2d4add880e50f470e4cb0687e20d16e6e926d
Matt McEuen 6 months ago
parent
commit
178193be84

+ 20
- 0
charts/apiserver/templates/configmap-etc.yaml View File

@@ -17,6 +17,21 @@ limitations under the License.
17 17
 {{- if .Values.manifests.configmap_etc }}
18 18
 {{- $envAll := . }}
19 19
 
20
+{{/* This slightly involved merge of AC config files into the anchor
21
+     files uses HTK merge, as straighforward appends result in duplicates. */}}
22
+{{- $_ := set .Values "_ac_files_to_copy" list }}
23
+{{- range $key, $val := .Values.conf.admission_controllers }}
24
+  {{- $source := printf "/tmp/etc/%s" $key }}
25
+  {{- $dest := printf "/etc/kubernetes/apiserver/%s" $key }}
26
+  {{- $file_to_copy := dict "source" $source "dest" $dest }}
27
+  {{- $ac_files_to_copy := append $.Values._ac_files_to_copy $file_to_copy }}
28
+  {{- $_ := set $.Values "_ac_files_to_copy" $ac_files_to_copy }}
29
+{{- end }}
30
+{{ $all_files_to_copy := dict }}
31
+{{ $_ := set $all_files_to_copy "values" (tuple .Values.anchor.files_to_copy .Values._ac_files_to_copy) }}
32
+{{ $_ := $all_files_to_copy | include "helm-toolkit.utils.merge" }}
33
+{{ $_ := set .Values.anchor "files_to_copy" $all_files_to_copy.result }}
34
+
20 35
 ---
21 36
 apiVersion: v1
22 37
 kind: ConfigMap
@@ -27,4 +42,9 @@ data:
27 42
 {{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
28 43
   kubeconfig.yaml: |+
29 44
 {{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
45
+{{/* Dynamically add config files for admission controllers */}}
46
+{{ range $key, $val := .Values.conf.admission_controllers }}
47
+  {{ $key }}: |+
48
+{{ toYaml $val | indent 4 }}
49
+{{ end }}
30 50
 {{- end }}

+ 1
- 0
charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl View File

@@ -62,6 +62,7 @@ spec:
62 62
         - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
63 63
         - --allow-privileged=true
64 64
         - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
65
+        - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
65 66
 
66 67
       ports:
67 68
         - containerPort: {{ .Values.network.kubernetes_apiserver.port }}

+ 24
- 3
charts/apiserver/values.yaml View File

@@ -55,20 +55,41 @@ anchor:
55 55
       dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml
56 56
     - source: /tmp/etc/kubeconfig.yaml
57 57
       dest: /etc/kubernetes/apiserver/kubeconfig.yaml
58
+    # Note: config files for admission controllers are added to this dynamically
58 59
 
59 60
 command_prefix:
60 61
   - /apiserver
61 62
   - --authorization-mode=Node,RBAC
62
-  - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
63
-  - --apiserver-count=3
63
+  - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
64 64
   - --service-cluster-ip-range=10.96.0.0/16
65
-  - --v=5
65
+  - --endpoint-reconciler-type=lease
66
+  # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
67
+  - --repair-malformed-updates=false
66 68
 
67 69
 apiserver:
68 70
   host_etc_path: /etc/kubernetes/apiserver
69 71
   etcd:
70 72
     endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local
71 73
 
74
+conf:
75
+  # Admission controllers config files are generated dynamically based on the
76
+  # config below, as they they are specific to particular ACs that may be
77
+  # configured by the operator (or added by k8s in the future).
78
+  admission_controllers:
79
+    eventconfig.yaml:
80
+      kind: Configuration
81
+      apiVersion: eventratelimit.admission.k8s.io/v1alpha1
82
+      limits:
83
+      - type: Server
84
+        qps: 100
85
+        burst: 1000
86
+    acconfig.yaml:
87
+      kind: AdmissionConfiguration
88
+      apiVersion: apiserver.k8s.io/v1alpha1
89
+      plugins:
90
+      - name: EventRateLimit
91
+        path: eventconfig.yaml
92
+
72 93
 network:
73 94
   kubernetes_apiserver:
74 95
     ingress:

+ 1
- 1
examples/basic/Genesis.yaml View File

@@ -14,7 +14,7 @@ data:
14 14
     command_prefix:
15 15
       - /apiserver
16 16
       - --authorization-mode=Node,RBAC
17
-      - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction
17
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
18 18
       - --service-cluster-ip-range=10.96.0.0/16
19 19
       - --endpoint-reconciler-type=lease
20 20
       # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11

+ 1
- 1
examples/basic/armada-resources.yaml View File

@@ -743,7 +743,7 @@ data:
743 743
     command_prefix:
744 744
       - /apiserver
745 745
       - --authorization-mode=Node,RBAC
746
-      - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction
746
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
747 747
       - --service-cluster-ip-range=10.96.0.0/16
748 748
       - --endpoint-reconciler-type=lease
749 749
       # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11

+ 6
- 0
promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml View File

@@ -0,0 +1,6 @@
1
+---
2
+kind: AdmissionConfiguration
3
+apiVersion: apiserver.k8s.io/v1alpha1
4
+plugins:
5
+- name: EventRateLimit
6
+  path: eventconfig.yaml

+ 7
- 0
promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml View File

@@ -0,0 +1,7 @@
1
+---
2
+kind: Configuration
3
+apiVersion: eventratelimit.admission.k8s.io/v1alpha1
4
+limits:
5
+- type: Server
6
+  qps: 100
7
+  burst: 1000

+ 1
- 4
promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml View File

@@ -122,8 +122,6 @@ spec:
122 122
       - "{{ argument }}"
123 123
       {%- endfor %}
124 124
       - --advertise-address={{ config['Genesis:ip'] }}
125
-      - --authorization-mode=Node,RBAC
126
-      - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
127 125
       - --anonymous-auth=false
128 126
       - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
129 127
       - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
@@ -132,15 +130,14 @@ spec:
132 130
       - --insecure-port=8080
133 131
       - --secure-port=6444
134 132
       - --bind-address=0.0.0.0
135
-      - --runtime-config=batch/v2alpha1=true
136 133
       - --allow-privileged=true
137 134
       - --etcd-servers=https://localhost:12379
138 135
       - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
139 136
       - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
140 137
       - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
141
-      - --service-cluster-ip-range={{ config['KubernetesNetwork:kubernetes.service_cidr'] }}
142 138
       - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
143 139
       - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
140
+      - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
144 141
       - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
145 142
       - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
146 143
     env:

+ 1
- 4
promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml View File

@@ -20,8 +20,6 @@ spec:
20 20
         - "{{ argument }}"
21 21
         {%- endfor %}
22 22
         - --advertise-address={{ config['Genesis:ip'] }}
23
-        - --authorization-mode=Node,RBAC
24
-        - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
25 23
         - --anonymous-auth=false
26 24
         - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
27 25
         - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
@@ -30,15 +28,14 @@ spec:
30 28
         - --insecure-port=0
31 29
         - --bind-address=0.0.0.0
32 30
         - --secure-port=6443
33
-        - --runtime-config=batch/v2alpha1=true
34 31
         - --allow-privileged=true
35 32
         - --etcd-servers=https://localhost:2379
36 33
         - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
37 34
         - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
38 35
         - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
39
-        - --service-cluster-ip-range={{ config['KubernetesNetwork:kubernetes.service_cidr'] }}
40 36
         - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
41 37
         - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
38
+        - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
42 39
         - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
43 40
         - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
44 41
       volumeMounts:

Loading…
Cancel
Save