Browse Source

Refactor API server

This change accomplishes 2 primary things:
1. It generalizes work to enable the EventRateLimit admission plugin.
2. It restructures the anchor so that during an upgrade an "old" anchor
   does not try to coordinate the injection of "new" data from
   configmaps/secrets.

It also includes these ancillary changes:
* Clean up apiserver argument specification in the chart.
* De-duplicate and realign apiserver arguments in bootstrapping templates.

It has the side effects of:
* Adding a new field, ".apiserver.arguments" to the Genesis config,
  which will be the preferred way to configure bootstrapping apiservers
  going forward (in lieu of command_prefix).

Change-Id: I33cfe80ee8e29cd79e479a7985e3c098a2288fda
Mark Burnett 4 months ago
parent
commit
04da7585ff

+ 41
- 13
charts/apiserver/templates/bin/_anchor.tpl View File

@@ -15,26 +15,54 @@
15 15
 
16 16
 set -x
17 17
 
18
-compare_copy_files() {
18
+snapshot_files() {
19
+    SNAPSHOT_DIR=${1}
20
+    {{ range $dest, $source := .Values.const.files_to_copy }}
21
+    mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
22
+    cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
23
+    {{- end }}
24
+    {{ range $key, $val := .Values.conf }}
25
+    cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
26
+    {{- end }}
27
+}
19 28
 
20
-    {{range .Values.anchor.files_to_copy}}
21
-    if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
22
-        mkdir -p $(dirname /host{{ .dest }})
23
-        cp {{ .source }} /host{{ .dest }}
24
-        chmod go-rwx /host{{ .dest }}
29
+compare_copy_files() {
30
+    SNAPSHOT_DIR=${1}
31
+    {{ range $dest, $source := .Values.const.files_to_copy }}
32
+    SRC="${SNAPSHOT_DIR}{{ $dest }}"
33
+    DEST="/host{{ $dest }}"
34
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
35
+        mkdir -p $(dirname "${DEST}")
36
+        cp "${SRC}" "${DEST}"
37
+        chmod go-rwx "${DEST}"
25 38
     fi
26
-    {{end}}
39
+    {{- end}}
40
+    {{ range $key, $val := .Values.conf }}
41
+    SRC="${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
42
+    DEST="/host/etc/kubernetes/apiserver/{{ $val.file }}"
43
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
44
+        mkdir -p $(dirname "${DEST}")
45
+        cp "${SRC}" "${DEST}"
46
+        chmod go-rwx "${DEST}"
47
+    fi
48
+    {{- end }}
27 49
 }
28 50
 
29 51
 cleanup() {
30
-
31
-    {{range .Values.anchor.files_to_copy}}
32
-    rm -f /host{{ .dest }}
33
-    {{end}}
52
+    {{- range $dest, $source := .Values.const.files_to_copy }}
53
+    rm -f "/host{{ $dest }}"
54
+    {{- end }}
55
+    {{  range $key, $val := .Values.conf }}
56
+    rm -f "/host/{{ $val.file }}"
57
+    {{- end }}
34 58
 }
35 59
 
36
-while true; do
37 60
 
61
+SNAPSHOT_DIR=$(mktemp -d)
62
+
63
+snapshot_files "${SNAPSHOT_DIR}"
64
+
65
+while true; do
38 66
     if [ -e /tmp/stop ]; then
39 67
         echo Stopping
40 68
         cleanup
@@ -43,7 +71,7 @@ while true; do
43 71
 
44 72
     # Compare and replace files on Genesis host if needed
45 73
     # Copy files to other master nodes
46
-    compare_copy_files
74
+    compare_copy_files "${SNAPSHOT_DIR}"
47 75
 
48 76
     sleep {{ .Values.anchor.period }}
49 77
 done

+ 7
- 22
charts/apiserver/templates/configmap-etc.yaml View File

@@ -17,34 +17,19 @@ limitations under the License.
17 17
 {{- if .Values.manifests.configmap_etc }}
18 18
 {{- $envAll := . }}
19 19
 
20
-{{/* This slightly involved merge of AC config files into the anchor
21
-     files uses HTK merge, as straighforward appends result in duplicates. */}}
22
-{{- $_ := set .Values "_ac_files_to_copy" list }}
23
-{{- range $key, $val := .Values.conf.admission_controllers }}
24
-  {{- $source := printf "/tmp/etc/%s" $key }}
25
-  {{- $dest := printf "/etc/kubernetes/apiserver/%s" $key }}
26
-  {{- $file_to_copy := dict "source" $source "dest" $dest }}
27
-  {{- $ac_files_to_copy := append $.Values._ac_files_to_copy $file_to_copy }}
28
-  {{- $_ := set $.Values "_ac_files_to_copy" $ac_files_to_copy }}
29
-{{- end }}
30
-{{ $all_files_to_copy := dict }}
31
-{{ $_ := set $all_files_to_copy "values" (tuple .Values.anchor.files_to_copy .Values._ac_files_to_copy) }}
32
-{{ $_ := $all_files_to_copy | include "helm-toolkit.utils.merge" }}
33
-{{ $_ := set .Values.anchor "files_to_copy" $all_files_to_copy.result }}
34
-
35 20
 ---
36 21
 apiVersion: v1
37 22
 kind: ConfigMap
38 23
 metadata:
39 24
   name: {{ .Values.service.name }}-etc
40 25
 data:
41
-  kubernetes-apiserver.yaml: |+
26
+  kubernetes-apiserver.yaml: |
42 27
 {{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
43
-  kubeconfig.yaml: |+
28
+  kubeconfig.yaml: |
44 29
 {{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
45
-{{/* Dynamically add config files for admission controllers */}}
46
-{{ range $key, $val := .Values.conf.admission_controllers }}
47
-  {{ $key }}: |+
48
-{{ toYaml $val | indent 4 }}
49
-{{ end }}
30
+{{/* Dynamically added config files */}}
31
+{{- range $key, $val := .Values.conf }}
32
+  {{ $val.file }}: |
33
+{{ toYaml $val.content | indent 4 }}
34
+{{- end }}
50 35
 {{- end }}

+ 15
- 20
charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl View File

@@ -42,30 +42,25 @@ spec:
42 42
               fieldPath: spec.nodeName
43 43
         - name: KUBECONFIG
44 44
           value: /etc/kubernetes/apiserver/kubeconfig.yaml
45
+        - name: APISERVER_PORT
46
+          value: {{ .Values.network.kubernetes_apiserver.port | quote }}
47
+        - name: ETCD_ENDPOINTS
48
+          value: {{ .Values.apiserver.etcd.endpoints | quote }}
45 49
 
46 50
       command:
47
-        {{- range .Values.command_prefix }}
51
+        {{- range .Values.const.command_prefix }}
48 52
         - {{ . }}
49 53
         {{- end }}
50
-        - --advertise-address=$(POD_IP)
51
-        - --anonymous-auth=false
52
-        - --bind-address=0.0.0.0
53
-        - --secure-port={{ .Values.network.kubernetes_apiserver.port }}
54
-        - --insecure-port=0
55
-        - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
56
-        - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
57
-        - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
58
-        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
59
-        - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
60
-        - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
61
-        - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
62
-        - --etcd-servers={{ .Values.apiserver.etcd.endpoints }}
63
-        - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
64
-        - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
65
-        - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
66
-        - --allow-privileged=true
67
-        - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
68
-        - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
54
+        {{- range .Values.apiserver.arguments }}
55
+        - {{ . }}
56
+        {{- end }}
57
+        {{- range $key, $val := .Values.conf }}
58
+        {{- if hasKey $val "command_options" }}
59
+        {{- range $val.command_options }}
60
+        - {{ . }}
61
+        {{- end }}
62
+        {{- end }}
63
+        {{- end }}
69 64
 
70 65
       ports:
71 66
         - containerPort: {{ .Values.network.kubernetes_apiserver.port }}

+ 87
- 56
charts/apiserver/values.yaml View File

@@ -14,6 +14,45 @@
14 14
 
15 15
 release_group: null
16 16
 
17
+# NOTE(mark-burnett): These values are not really configurable -- they live
18
+# here to keep the templates cleaner.
19
+const:
20
+  command_prefix:
21
+    - /apiserver
22
+    - --advertise-address=$(POD_IP)
23
+    - --allow-privileged=true
24
+    - --anonymous-auth=false
25
+    - --bind-address=0.0.0.0
26
+    - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
27
+    - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
28
+    - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
29
+    - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
30
+    - --etcd-servers=$(ETCD_ENDPOINTS)
31
+    - --insecure-port=0
32
+    - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
33
+    - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
34
+    - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
35
+    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
36
+    - --secure-port=$(APISERVER_PORT)
37
+    - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
38
+    - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
39
+    - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
40
+
41
+  files_to_copy:
42
+    # NOTE(mark-burnett): These are (host dest): (container source) pairs
43
+    /etc/kubernetes/apiserver/kubeconfig.yaml: /tmp/etc/kubeconfig.yaml
44
+    /etc/kubernetes/apiserver/pki/apiserver-key.pem: /keys/apiserver-key.pem
45
+    /etc/kubernetes/apiserver/pki/apiserver.pem: /certs/apiserver.pem
46
+    /etc/kubernetes/apiserver/pki/cluster-ca.pem: /certs/cluster-ca.pem
47
+    /etc/kubernetes/apiserver/pki/etcd-client-ca.pem: /certs/etcd-client-ca.pem
48
+    /etc/kubernetes/apiserver/pki/etcd-client-key.pem: /keys/etcd-client-key.pem
49
+    /etc/kubernetes/apiserver/pki/etcd-client.pem: /certs/etcd-client.pem
50
+    /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem: /certs/kubelet-client-ca.pem
51
+    /etc/kubernetes/apiserver/pki/kubelet-client-key.pem: /keys/kubelet-client-key.pem
52
+    /etc/kubernetes/apiserver/pki/kubelet-client.pem: /certs/kubelet-client.pem
53
+    /etc/kubernetes/apiserver/pki/service-account.pub: /certs/service-account.pub
54
+    /etc/kubernetes/manifests/kubernetes-apiserver.yaml: /tmp/etc/kubernetes-apiserver.yaml
55
+
17 56
 images:
18 57
   tags:
19 58
     anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.11
@@ -30,65 +69,58 @@ anchor:
30 69
   kubelet:
31 70
     manifest_path: /etc/kubernetes/manifests
32 71
   period: 15
33
-  files_to_copy:
34
-    - source: /certs/apiserver.pem
35
-      dest: /etc/kubernetes/apiserver/pki/apiserver.pem
36
-    - source: /certs/kubelet-client.pem
37
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client.pem
38
-    - source: /certs/kubelet-client-ca.pem
39
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
40
-    - source: /certs/cluster-ca.pem
41
-      dest: /etc/kubernetes/apiserver/pki/cluster-ca.pem
42
-    - source: /certs/etcd-client-ca.pem
43
-      dest: /etc/kubernetes/apiserver/pki/etcd-client-ca.pem
44
-    - source: /certs/etcd-client.pem
45
-      dest: /etc/kubernetes/apiserver/pki/etcd-client.pem
46
-    - source: /certs/service-account.pub
47
-      dest: /etc/kubernetes/apiserver/pki/service-account.pub
48
-    - source: /keys/apiserver-key.pem
49
-      dest: /etc/kubernetes/apiserver/pki/apiserver-key.pem
50
-    - source: /keys/kubelet-client-key.pem
51
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client-key.pem
52
-    - source: /keys/etcd-client-key.pem
53
-      dest: /etc/kubernetes/apiserver/pki/etcd-client-key.pem
54
-    - source: /tmp/etc/kubernetes-apiserver.yaml
55
-      dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml
56
-    - source: /tmp/etc/kubeconfig.yaml
57
-      dest: /etc/kubernetes/apiserver/kubeconfig.yaml
58
-    # Note: config files for admission controllers are added to this dynamically
59 72
 
60
-command_prefix:
61
-  - /apiserver
62
-  - --authorization-mode=Node,RBAC
63
-  - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
64
-  - --service-cluster-ip-range=10.96.0.0/16
65
-  - --endpoint-reconciler-type=lease
66
-  # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
67
-  - --repair-malformed-updates=false
73
+conf:
74
+# Uncomment any of the below to enable the file placement and associated apiserver
75
+# command line options
76
+#
77
+#  acconfig:
78
+#    file: acconfig.yaml
79
+#    command_options:
80
+#      - '--admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml'
81
+#      - '--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit'
82
+#    content:
83
+#      kind: AdmissionConfiguration
84
+#      apiVersion: apiserver.k8s.io/v1alpha1
85
+#      plugins:
86
+#        - name: EventRateLimit
87
+#          path: eventconfig.yaml
88
+#  eventconfig:
89
+#    file: eventconfig.yaml
90
+#    command_options:
91
+#      - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml'
92
+#    content:
93
+#      kind: Configuration
94
+#      apiVersion: eventratelimit.admission.k8s.io/v1alpha1
95
+#      limits:
96
+#        - type: Server
97
+#          qps: 1000
98
+#          burst: 10000
99
+#  encryption_provider:
100
+#    file: encryption_provider.yaml
101
+#    command_option: ''
102
+#    content:
103
+#      kind: EncryptionConfig
104
+#      apiVersion: v1
105
+#      resources:
106
+#        - resources:
107
+#            - 'secrets'
108
+#          providers:
109
+#            - identity: {}
68 110
 
69 111
 apiserver:
70
-  host_etc_path: /etc/kubernetes/apiserver
112
+  arguments:
113
+    - --authorization-mode=Node,RBAC
114
+    - --service-cluster-ip-range=10.96.0.0/16
115
+    - --endpoint-reconciler-type=lease
116
+    - --feature-gates=PodShareProcessNamespace=true
117
+    # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
118
+    - --repair-malformed-updates=false
119
+    - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction
120
+    - --v=3
71 121
   etcd:
72 122
     endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local
73
-
74
-conf:
75
-  # Admission controllers config files are generated dynamically based on the
76
-  # config below, as they are specific to particular ACs that may be
77
-  # configured by the operator (or added by k8s in the future).
78
-  admission_controllers:
79
-    eventconfig.yaml:
80
-      kind: Configuration
81
-      apiVersion: eventratelimit.admission.k8s.io/v1alpha1
82
-      limits:
83
-      - type: Server
84
-        qps: 100
85
-        burst: 1000
86
-    acconfig.yaml:
87
-      kind: AdmissionConfiguration
88
-      apiVersion: apiserver.k8s.io/v1alpha1
89
-      plugins:
90
-      - name: EventRateLimit
91
-        path: eventconfig.yaml
123
+  host_etc_path: /etc/kubernetes/apiserver
92 124
 
93 125
 network:
94 126
   kubernetes_apiserver:
@@ -130,7 +162,6 @@ secrets:
130 162
       cert: null
131 163
       key: null
132 164
 
133
-
134 165
 # typically overriden by environmental
135 166
 # values, but should include all endpoints
136 167
 # required by this chart
@@ -170,7 +201,7 @@ pod:
170 201
     upgrades:
171 202
       daemonsets:
172 203
         pod_replacement_strategy: RollingUpdate
173
-        kubernetes_apiserver:
204
+        kubernetes-apiserver-anchor:
174 205
           enabled: false
175 206
           min_ready_seconds: 0
176 207
           max_unavailable: 1

+ 22
- 3
examples/basic/Genesis.yaml View File

@@ -11,15 +11,16 @@ data:
11 11
   hostname: n0
12 12
   ip: 192.168.77.10
13 13
   apiserver:
14
-    command_prefix:
15
-      - /apiserver
14
+    arguments:
16 15
       - --authorization-mode=Node,RBAC
17
-      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
16
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
18 17
       - --service-cluster-ip-range=10.96.0.0/16
19 18
       - --endpoint-reconciler-type=lease
20 19
       - --feature-gates=PodShareProcessNamespace=true
21 20
       # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
22 21
       - --repair-malformed-updates=false
22
+      - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
23
+      - --v=3
23 24
   armada:
24 25
     target_manifest: cluster-bootstrap
25 26
   labels:
@@ -45,4 +46,22 @@ data:
45 46
     - path: /var/lib/anchor/calico-etcd-bootstrap
46 47
       content: "# placeholder for triggering calico etcd bootstrapping"
47 48
       mode: 0644
49
+    # NOTE(mark-burnett): These are referenced by the apiserver arguments above.
50
+    - path: /etc/genesis/apiserver/acconfig.yaml
51
+      mode: 0444
52
+      content: |
53
+        kind: AdmissionConfiguration
54
+        apiVersion: apiserver.k8s.io/v1alpha1
55
+        plugins:
56
+          - name: EventRateLimit
57
+            path: eventconfig.yaml
58
+    - path: /etc/genesis/apiserver/eventconfig.yaml
59
+      mode: 0444
60
+      content: |
61
+        kind: Configuration
62
+        apiVersion: eventratelimit.admission.k8s.io/v1alpha1
63
+        limits:
64
+          - type: Server
65
+            qps: 1000
66
+            burst: 10000
48 67
 ...

+ 0
- 9
examples/basic/armada-resources.yaml View File

@@ -719,15 +719,6 @@ data:
719 719
   upgrade:
720 720
     no_hooks: true
721 721
   values:
722
-    command_prefix:
723
-      - /apiserver
724
-      - --authorization-mode=Node,RBAC
725
-      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
726
-      - --service-cluster-ip-range=10.96.0.0/16
727
-      - --endpoint-reconciler-type=lease
728
-      - --feature-gates=PodShareProcessNamespace=true
729
-      # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
730
-      - --repair-malformed-updates=false
731 722
     apiserver:
732 723
       etcd:
733 724
         endpoints: https://127.0.0.1:2378

+ 1
- 1
promenade/config.py View File

@@ -241,7 +241,7 @@ class Configuration:
241 241
 
242 242
     def bootstrap_apiserver_prefix(self):
243 243
         return self.get_path('Genesis:apiserver.command_prefix',
244
-                             ['/apiserver', '--apiserver-count=2', '--v=5'])
244
+                             ['/apiserver'])
245 245
 
246 246
 
247 247
 def _matches_filter(document, *, schema, labels, name):

+ 4
- 0
promenade/schemas/Genesis.yaml View File

@@ -71,6 +71,10 @@ data:
71 71
           type: array
72 72
           items:
73 73
             type: string
74
+        arguments:
75
+          type: array
76
+          items:
77
+            type: string
74 78
       additionalProperties: false
75 79
 
76 80
     files:

+ 18
- 0
promenade/templates/include/genesis-apiserver.yaml View File

@@ -0,0 +1,18 @@
1
+        - --advertise-address={{ config['Genesis:ip'] }}
2
+        - --allow-privileged=true
3
+        - --anonymous-auth=false
4
+        - --bind-address=0.0.0.0
5
+        - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
6
+        - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
7
+        - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
8
+        - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
9
+        - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
10
+        - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
11
+        - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
12
+        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
13
+        - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
14
+        - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
15
+        - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
16
+        {%- for argument in config.get_path('Genesis:apiserver.arguments', []) %}
17
+        - "{{ argument }}"
18
+        {%- endfor %}

+ 0
- 6
promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml View File

@@ -1,6 +0,0 @@
1
----
2
-kind: AdmissionConfiguration
3
-apiVersion: apiserver.k8s.io/v1alpha1
4
-plugins:
5
-- name: EventRateLimit
6
-  path: eventconfig.yaml

+ 0
- 7
promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml View File

@@ -1,7 +0,0 @@
1
----
2
-kind: Configuration
3
-apiVersion: eventratelimit.admission.k8s.io/v1alpha1
4
-limits:
5
-- type: Server
6
-  qps: 100
7
-  burst: 1000

+ 120
- 136
promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml View File

@@ -11,146 +11,130 @@ spec:
11 11
   dnsPolicy: Default
12 12
   hostNetwork: true
13 13
   containers:
14
-  - env:
15
-    - name: TILLER_NAMESPACE
16
-      value: kube-system
17
-    image: {{ config['Genesis:images.helm.tiller'] }}
18
-    command:
19
-      - /tiller
20
-      - -logtostderr
21
-      - -v
22
-      - "99"
23
-    imagePullPolicy: IfNotPresent
24
-    livenessProbe:
25
-      failureThreshold: 3
26
-      httpGet:
27
-        path: /liveness
28
-        port: 44135
29
-        scheme: HTTP
30
-      initialDelaySeconds: 1
31
-      periodSeconds: 10
32
-      successThreshold: 1
33
-      timeoutSeconds: 1
34
-    name: tiller
35
-    ports:
36
-    - containerPort: 44134
14
+    - env:
15
+      - name: TILLER_NAMESPACE
16
+        value: kube-system
17
+      image: {{ config['Genesis:images.helm.tiller'] }}
18
+      command:
19
+        - /tiller
20
+        - -logtostderr
21
+        - -v
22
+        - "99"
23
+      imagePullPolicy: IfNotPresent
24
+      livenessProbe:
25
+        failureThreshold: 3
26
+        httpGet:
27
+          path: /liveness
28
+          port: 44135
29
+          scheme: HTTP
30
+        initialDelaySeconds: 1
31
+        periodSeconds: 10
32
+        successThreshold: 1
33
+        timeoutSeconds: 1
37 34
       name: tiller
38
-      protocol: TCP
39
-    readinessProbe:
40
-      failureThreshold: 3
41
-      httpGet:
42
-        path: /readiness
43
-        port: 44135
44
-        scheme: HTTP
45
-      initialDelaySeconds: 1
46
-      periodSeconds: 10
47
-      successThreshold: 1
48
-      timeoutSeconds: 1
49
-    resources: {}
50
-    terminationMessagePath: /dev/termination-log
51
-    terminationMessagePolicy: File
52
-  - name: armada
53
-    image: {{ config['Genesis:images.armada'] }}
54
-    securityContext:
55
-      runAsUser: 0
56
-    command:
57
-      - /bin/bash
58
-      - -c
59
-      - |-
60
-        set -x
35
+      ports:
36
+      - containerPort: 44134
37
+        name: tiller
38
+        protocol: TCP
39
+      readinessProbe:
40
+        failureThreshold: 3
41
+        httpGet:
42
+          path: /readiness
43
+          port: 44135
44
+          scheme: HTTP
45
+        initialDelaySeconds: 1
46
+        periodSeconds: 10
47
+        successThreshold: 1
48
+        timeoutSeconds: 1
49
+      resources: {}
50
+      terminationMessagePath: /dev/termination-log
51
+      terminationMessagePolicy: File
52
+    - name: armada
53
+      image: {{ config['Genesis:images.armada'] }}
54
+      securityContext:
55
+        runAsUser: 0
56
+      command:
57
+        - /bin/bash
58
+        - -c
59
+        - |-
60
+          set -x
61 61
 
62
-        while true; do
63
-            sleep 10
64
-            if armada \
65
-                    apply \
66
-                    --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
67
-                    --tiller-host 127.0.0.1 \
68
-                    /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
69
-                break
70
-            fi
71
-        done
62
+          while true; do
63
+              sleep 10
64
+              if armada \
65
+                      apply \
66
+                      --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
67
+                      --tiller-host 127.0.0.1 \
68
+                      /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
69
+                  break
70
+              fi
71
+          done
72
+          touch /ipc/armada-done
73
+          sleep 10000
74
+      env:
75
+        - name: ARMADA_LOGFILE
76
+          value: /tmp/log/bootstrap-armada.log
77
+  {%- if config['KubernetesNetwork:proxy.url'] is defined %}
78
+        - name: HTTP_PROXY
79
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
80
+        - name: HTTPS_PROXY
81
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
82
+        - name: NO_PROXY
83
+          value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
84
+        - name: http_proxy
85
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
86
+        - name: https_proxy
87
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
88
+        - name: no_proxy
89
+          value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
90
+  {%- endif %}
91
+      volumeMounts:
92
+        - name: assets
93
+          mountPath: /etc/genesis/armada/assets
94
+        - name: auth
95
+          mountPath: /root/.kube
96
+        - name: ipc
97
+          mountPath: /ipc
98
+        - name: log
99
+          mountPath: /tmp/log
100
+    - name: monitor
101
+      image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
102
+      command:
103
+        - /bin/sh
104
+        - -c
105
+        - |-
106
+          set -x
72 107
 
73
-        touch /ipc/armada-done
74
-        sleep 10000
75
-    env:
76
-      - name: ARMADA_LOGFILE
77
-        value: /tmp/log/bootstrap-armada.log
78
-{%- if config['KubernetesNetwork:proxy.url'] is defined %}
79
-      - name: HTTP_PROXY
80
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
81
-      - name: HTTPS_PROXY
82
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
83
-      - name: NO_PROXY
84
-        value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
85
-      - name: http_proxy
86
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
87
-      - name: https_proxy
88
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
89
-      - name: no_proxy
90
-        value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
91
-{%- endif %}
92
-    volumeMounts:
93
-      - name: assets
94
-        mountPath: /etc/genesis/armada/assets
95
-      - name: auth
96
-        mountPath: /root/.kube
97
-      - name: ipc
98
-        mountPath: /ipc
99
-      - name: log
100
-        mountPath: /tmp/log
101
-  - name: monitor
102
-    image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
103
-    command:
104
-      - /bin/sh
105
-      - -c
106
-      - |-
107
-        set -x
108
+          while ! [ -e /ipc/armada-done ]; do
109
+            sleep 5
110
+          done
108 111
 
109
-        while ! [ -e /ipc/armada-done ]; do
110
-          sleep 5
111
-        done
112
-
113
-        rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml
114
-        sleep 10000
115
-    volumeMounts:
116
-      - name: ipc
117
-        mountPath: /ipc
118
-      - name: manifest
119
-        mountPath: /etc/kubernetes/manifests
120
-  - name: kubectl-apiserver
121
-    image: {{ config['Genesis:images.kubernetes.apiserver'] }}
122
-    command:
123
-      {%- for argument in config.bootstrap_apiserver_prefix() %}
124
-      - "{{ argument }}"
125
-      {%- endfor %}
126
-      - --advertise-address={{ config['Genesis:ip'] }}
127
-      - --anonymous-auth=false
128
-      - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
129
-      - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
130
-      - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem
131
-      - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem
132
-      - --insecure-port=8080
133
-      - --secure-port=6444
134
-      - --bind-address=0.0.0.0
135
-      - --allow-privileged=true
136
-      - --etcd-servers=https://localhost:12379
137
-      - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
138
-      - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
139
-      - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
140
-      - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
141
-      - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
142
-      - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
143
-      - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
144
-      - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
145
-    env:
146
-      - name: KUBECONFIG
147
-        value: /etc/kubernetes/admin/config
148
-    volumeMounts:
149
-      - name: auth
150
-        mountPath: /etc/kubernetes/admin
151
-      - name: config
152
-        mountPath: /etc/kubernetes/apiserver
153
-        readOnly: true
112
+          rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml
113
+          sleep 10000
114
+      volumeMounts:
115
+        - name: ipc
116
+          mountPath: /ipc
117
+        - name: manifest
118
+          mountPath: /etc/kubernetes/manifests
119
+    - name: kubectl-apiserver
120
+      image: {{ config['Genesis:images.kubernetes.apiserver'] }}
121
+      command:
122
+        {%- for argument in config.bootstrap_apiserver_prefix() %}
123
+        - "{{ argument }}"
124
+        {%- endfor %}
125
+{% include "genesis-apiserver.yaml" with context %}
126
+        - --etcd-servers=https://localhost:12379
127
+        - --insecure-port=8080
128
+        - --secure-port=6444
129
+      env:
130
+        - name: KUBECONFIG
131
+          value: /etc/kubernetes/admin/config
132
+      volumeMounts:
133
+        - name: auth
134
+          mountPath: /etc/kubernetes/admin
135
+        - name: config
136
+          mountPath: /etc/kubernetes/apiserver
137
+          readOnly: true
154 138
   volumes:
155 139
     - name: assets
156 140
       hostPath:

+ 2
- 17
promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml View File

@@ -19,25 +19,10 @@ spec:
19 19
         {%- for argument in config.bootstrap_apiserver_prefix() %}
20 20
         - "{{ argument }}"
21 21
         {%- endfor %}
22
-        - --advertise-address={{ config['Genesis:ip'] }}
23
-        - --anonymous-auth=false
24
-        - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
25
-        - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
26
-        - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
27
-        - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
22
+{% include "genesis-apiserver.yaml" with context %}
23
+        - --etcd-servers=https://localhost:2379
28 24
         - --insecure-port=0
29
-        - --bind-address=0.0.0.0
30 25
         - --secure-port=6443
31
-        - --allow-privileged=true
32
-        - --etcd-servers=https://localhost:2379
33
-        - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
34
-        - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
35
-        - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
36
-        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
37
-        - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
38
-        - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
39
-        - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
40
-        - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
41 26
       volumeMounts:
42 27
         - name: config
43 28
           mountPath: /etc/kubernetes/apiserver

Loading…
Cancel
Save