Refactor API server

This change accomplishes 2 primary things:
1. It generalizes work to enable the EventRateLimit admission plugin.
2. It restructures the anchor so that during an upgrade an "old" anchor
   does not try to coordinate the injection of "new" data from
   configmaps/secrets.

It also includes these ancillary changes:
* Clean up apiserver argument specification in the chart.
* De-duplicate and realign apiserver arguments in bootstrapping templates.

It has the side effects of:
* Adding a new field, ".apiserver.arguments" to the Genesis config,
  which will be the preferred way to configure bootstrapping apiservers
  going forward (in lieu of command_prefix).

Change-Id: I33cfe80ee8e29cd79e479a7985e3c098a2288fda

charts/apiserver/templates/bin/_anchor.tpl

@@ -15,26 +15,54 @@
 
 set -x
 
-compare_copy_files() {
+snapshot_files() {
+    SNAPSHOT_DIR=${1}
+    {{ range $dest, $source := .Values.const.files_to_copy }}
+    mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
+    cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
+    {{- end }}
+    {{ range $key, $val := .Values.conf }}
+    cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
+    {{- end }}
+}
 
-    {{range .Values.anchor.files_to_copy}}
+compare_copy_files() {
-    if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
+    SNAPSHOT_DIR=${1}
-        mkdir -p $(dirname /host{{ .dest }})
+    {{ range $dest, $source := .Values.const.files_to_copy }}
-        cp {{ .source }} /host{{ .dest }}
+    SRC="${SNAPSHOT_DIR}{{ $dest }}"
-        chmod go-rwx /host{{ .dest }}
+    DEST="/host{{ $dest }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
     fi
-    {{end}}
+    {{- end}}
+    {{ range $key, $val := .Values.conf }}
+    SRC="${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
+    DEST="/host/etc/kubernetes/apiserver/{{ $val.file }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
+    fi
+    {{- end }}
 }
 
 cleanup() {
-
+    {{- range $dest, $source := .Values.const.files_to_copy }}
-    {{range .Values.anchor.files_to_copy}}
+    rm -f "/host{{ $dest }}"
-    rm -f /host{{ .dest }}
+    {{- end }}
-    {{end}}
+    {{  range $key, $val := .Values.conf }}
+    rm -f "/host/{{ $val.file }}"
+    {{- end }}
 }
 
-while true; do
 
+SNAPSHOT_DIR=$(mktemp -d)
+
+snapshot_files "${SNAPSHOT_DIR}"
+
+while true; do
     if [ -e /tmp/stop ]; then
         echo Stopping
         cleanup
@@ -43,7 +71,7 @@ while true; do
 
     # Compare and replace files on Genesis host if needed
     # Copy files to other master nodes
-    compare_copy_files
+    compare_copy_files "${SNAPSHOT_DIR}"
 
     sleep {{ .Values.anchor.period }}
 done

charts/apiserver/templates/configmap-etc.yaml

@@ -17,34 +17,19 @@ limitations under the License.
 {{- if .Values.manifests.configmap_etc }}
 {{- $envAll := . }}
 
-{{/* This slightly involved merge of AC config files into the anchor
-     files uses HTK merge, as straighforward appends result in duplicates. */}}
-{{- $_ := set .Values "_ac_files_to_copy" list }}
-{{- range $key, $val := .Values.conf.admission_controllers }}
-  {{- $source := printf "/tmp/etc/%s" $key }}
-  {{- $dest := printf "/etc/kubernetes/apiserver/%s" $key }}
-  {{- $file_to_copy := dict "source" $source "dest" $dest }}
-  {{- $ac_files_to_copy := append $.Values._ac_files_to_copy $file_to_copy }}
-  {{- $_ := set $.Values "_ac_files_to_copy" $ac_files_to_copy }}
-{{- end }}
-{{ $all_files_to_copy := dict }}
-{{ $_ := set $all_files_to_copy "values" (tuple .Values.anchor.files_to_copy .Values._ac_files_to_copy) }}
-{{ $_ := $all_files_to_copy | include "helm-toolkit.utils.merge" }}
-{{ $_ := set .Values.anchor "files_to_copy" $all_files_to_copy.result }}
-
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ .Values.service.name }}-etc
 data:
-  kubernetes-apiserver.yaml: |+
+  kubernetes-apiserver.yaml: |
 {{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  kubeconfig.yaml: |+
+  kubeconfig.yaml: |
 {{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{/* Dynamically add config files for admission controllers */}}
+{{/* Dynamically added config files */}}
-{{ range $key, $val := .Values.conf.admission_controllers }}
+{{- range $key, $val := .Values.conf }}
-  {{ $key }}: |+
+  {{ $val.file }}: |
-{{ toYaml $val | indent 4 }}
+{{ toYaml $val.content | indent 4 }}
-{{ end }}
+{{- end }}
 {{- end }}

charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl

@@ -42,30 +42,25 @@ spec:
               fieldPath: spec.nodeName
         - name: KUBECONFIG
           value: /etc/kubernetes/apiserver/kubeconfig.yaml
+        - name: APISERVER_PORT
+          value: {{ .Values.network.kubernetes_apiserver.port | quote }}
+        - name: ETCD_ENDPOINTS
+          value: {{ .Values.apiserver.etcd.endpoints | quote }}
 
       command:
-        {{- range .Values.command_prefix }}
+        {{- range .Values.const.command_prefix }}
         - {{ . }}
         {{- end }}
-        - --advertise-address=$(POD_IP)
+        {{- range .Values.apiserver.arguments }}
-        - --anonymous-auth=false
+        - {{ . }}
-        - --bind-address=0.0.0.0
+        {{- end }}
-        - --secure-port={{ .Values.network.kubernetes_apiserver.port }}
+        {{- range $key, $val := .Values.conf }}
-        - --insecure-port=0
+        {{- if hasKey $val "command_options" }}
-        - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+        {{- range $val.command_options }}
-        - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
+        - {{ . }}
-        - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
+        {{- end }}
-        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+        {{- end }}
-        - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+        {{- end }}
-        - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
-        - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
-        - --etcd-servers={{ .Values.apiserver.etcd.endpoints }}
-        - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
-        - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
-        - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
-        - --allow-privileged=true
-        - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
-        - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
 
       ports:
         - containerPort: {{ .Values.network.kubernetes_apiserver.port }}

charts/apiserver/values.yaml

@@ -14,6 +14,45 @@
 
 release_group: null
 
+# NOTE(mark-burnett): These values are not really configurable -- they live
+# here to keep the templates cleaner.
+const:
+  command_prefix:
+    - /apiserver
+    - --advertise-address=$(POD_IP)
+    - --allow-privileged=true
+    - --anonymous-auth=false
+    - --bind-address=0.0.0.0
+    - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+    - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
+    - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
+    - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
+    - --etcd-servers=$(ETCD_ENDPOINTS)
+    - --insecure-port=0
+    - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+    - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
+    - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
+    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+    - --secure-port=$(APISERVER_PORT)
+    - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
+    - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
+    - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
+
+  files_to_copy:
+    # NOTE(mark-burnett): These are (host dest): (container source) pairs
+    /etc/kubernetes/apiserver/kubeconfig.yaml: /tmp/etc/kubeconfig.yaml
+    /etc/kubernetes/apiserver/pki/apiserver-key.pem: /keys/apiserver-key.pem
+    /etc/kubernetes/apiserver/pki/apiserver.pem: /certs/apiserver.pem
+    /etc/kubernetes/apiserver/pki/cluster-ca.pem: /certs/cluster-ca.pem
+    /etc/kubernetes/apiserver/pki/etcd-client-ca.pem: /certs/etcd-client-ca.pem
+    /etc/kubernetes/apiserver/pki/etcd-client-key.pem: /keys/etcd-client-key.pem
+    /etc/kubernetes/apiserver/pki/etcd-client.pem: /certs/etcd-client.pem
+    /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem: /certs/kubelet-client-ca.pem
+    /etc/kubernetes/apiserver/pki/kubelet-client-key.pem: /keys/kubelet-client-key.pem
+    /etc/kubernetes/apiserver/pki/kubelet-client.pem: /certs/kubelet-client.pem
+    /etc/kubernetes/apiserver/pki/service-account.pub: /certs/service-account.pub
+    /etc/kubernetes/manifests/kubernetes-apiserver.yaml: /tmp/etc/kubernetes-apiserver.yaml
+
 images:
   tags:
     anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.11
@@ -30,65 +69,58 @@ anchor:
   kubelet:
     manifest_path: /etc/kubernetes/manifests
   period: 15
-  files_to_copy:
-    - source: /certs/apiserver.pem
-      dest: /etc/kubernetes/apiserver/pki/apiserver.pem
-    - source: /certs/kubelet-client.pem
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client.pem
-    - source: /certs/kubelet-client-ca.pem
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
-    - source: /certs/cluster-ca.pem
-      dest: /etc/kubernetes/apiserver/pki/cluster-ca.pem
-    - source: /certs/etcd-client-ca.pem
-      dest: /etc/kubernetes/apiserver/pki/etcd-client-ca.pem
-    - source: /certs/etcd-client.pem
-      dest: /etc/kubernetes/apiserver/pki/etcd-client.pem
-    - source: /certs/service-account.pub
-      dest: /etc/kubernetes/apiserver/pki/service-account.pub
-    - source: /keys/apiserver-key.pem
-      dest: /etc/kubernetes/apiserver/pki/apiserver-key.pem
-    - source: /keys/kubelet-client-key.pem
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client-key.pem
-    - source: /keys/etcd-client-key.pem
-      dest: /etc/kubernetes/apiserver/pki/etcd-client-key.pem
-    - source: /tmp/etc/kubernetes-apiserver.yaml
-      dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml
-    - source: /tmp/etc/kubeconfig.yaml
-      dest: /etc/kubernetes/apiserver/kubeconfig.yaml
-    # Note: config files for admission controllers are added to this dynamically
-
-command_prefix:
-  - /apiserver
-  - --authorization-mode=Node,RBAC
-  - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
-  - --service-cluster-ip-range=10.96.0.0/16
-  - --endpoint-reconciler-type=lease
-  # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
-  - --repair-malformed-updates=false
-
-apiserver:
-  host_etc_path: /etc/kubernetes/apiserver
-  etcd:
-    endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local
 
 conf:
-  # Admission controllers config files are generated dynamically based on the
+# Uncomment any of the below to enable the file placement and associated apiserver
-  # config below, as they are specific to particular ACs that may be
+# command line options
-  # configured by the operator (or added by k8s in the future).
+#
-  admission_controllers:
+#  acconfig:
-    eventconfig.yaml:
+#    file: acconfig.yaml
-      kind: Configuration
+#    command_options:
-      apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+#      - '--admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml'
-      limits:
+#      - '--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit'
-      - type: Server
+#    content:
-        qps: 100
+#      kind: AdmissionConfiguration
-        burst: 1000
+#      apiVersion: apiserver.k8s.io/v1alpha1
-    acconfig.yaml:
+#      plugins:
-      kind: AdmissionConfiguration
+#        - name: EventRateLimit
-      apiVersion: apiserver.k8s.io/v1alpha1
+#          path: eventconfig.yaml
-      plugins:
+#  eventconfig:
-      - name: EventRateLimit
+#    file: eventconfig.yaml
-        path: eventconfig.yaml
+#    command_options:
+#      - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml'
+#    content:
+#      kind: Configuration
+#      apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+#      limits:
+#        - type: Server
+#          qps: 1000
+#          burst: 10000
+#  encryption_provider:
+#    file: encryption_provider.yaml
+#    command_option: ''
+#    content:
+#      kind: EncryptionConfig
+#      apiVersion: v1
+#      resources:
+#        - resources:
+#            - 'secrets'
+#          providers:
+#            - identity: {}
+
+apiserver:
+  arguments:
+    - --authorization-mode=Node,RBAC
+    - --service-cluster-ip-range=10.96.0.0/16
+    - --endpoint-reconciler-type=lease
+    - --feature-gates=PodShareProcessNamespace=true
+    # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
+    - --repair-malformed-updates=false
+    - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction
+    - --v=3
+  etcd:
+    endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local
+  host_etc_path: /etc/kubernetes/apiserver
 
 network:
   kubernetes_apiserver:
@@ -130,7 +162,6 @@ secrets:
       cert: null
       key: null
 
-
 # typically overriden by environmental
 # values, but should include all endpoints
 # required by this chart
@@ -170,7 +201,7 @@ pod:
     upgrades:
       daemonsets:
         pod_replacement_strategy: RollingUpdate
-        kubernetes_apiserver:
+        kubernetes-apiserver-anchor:
           enabled: false
           min_ready_seconds: 0
           max_unavailable: 1

examples/basic/Genesis.yaml

@@ -11,15 +11,16 @@ data:
   hostname: n0
   ip: 192.168.77.10
   apiserver:
-    command_prefix:
+    arguments:
-      - /apiserver
       - --authorization-mode=Node,RBAC
-      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
       - --service-cluster-ip-range=10.96.0.0/16
       - --endpoint-reconciler-type=lease
       - --feature-gates=PodShareProcessNamespace=true
       # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
       - --repair-malformed-updates=false
+      - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
+      - --v=3
   armada:
     target_manifest: cluster-bootstrap
   labels:
@@ -45,4 +46,22 @@ data:
     - path: /var/lib/anchor/calico-etcd-bootstrap
       content: "# placeholder for triggering calico etcd bootstrapping"
       mode: 0644
+    # NOTE(mark-burnett): These are referenced by the apiserver arguments above.
+    - path: /etc/genesis/apiserver/acconfig.yaml
+      mode: 0444
+      content: |
+        kind: AdmissionConfiguration
+        apiVersion: apiserver.k8s.io/v1alpha1
+        plugins:
+          - name: EventRateLimit
+            path: eventconfig.yaml
+    - path: /etc/genesis/apiserver/eventconfig.yaml
+      mode: 0444
+      content: |
+        kind: Configuration
+        apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+        limits:
+          - type: Server
+            qps: 1000
+            burst: 10000
 ...

examples/basic/armada-resources.yaml

@@ -719,15 +719,6 @@ data:
   upgrade:
     no_hooks: true
   values:
-    command_prefix:
-      - /apiserver
-      - --authorization-mode=Node,RBAC
-      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
-      - --service-cluster-ip-range=10.96.0.0/16
-      - --endpoint-reconciler-type=lease
-      - --feature-gates=PodShareProcessNamespace=true
-      # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
-      - --repair-malformed-updates=false
     apiserver:
       etcd:
         endpoints: https://127.0.0.1:2378

promenade/config.py

@@ -241,7 +241,7 @@ class Configuration:
 
     def bootstrap_apiserver_prefix(self):
         return self.get_path('Genesis:apiserver.command_prefix',
-                             ['/apiserver', '--apiserver-count=2', '--v=5'])
+                             ['/apiserver'])
 
 
 def _matches_filter(document, *, schema, labels, name):

promenade/schemas/Genesis.yaml

@@ -71,6 +71,10 @@ data:
           type: array
           items:
             type: string
+        arguments:
+          type: array
+          items:
+            type: string
       additionalProperties: false
 
     files:

promenade/templates/include/genesis-apiserver.yaml

@@ -0,0 +1,18 @@
+        - --advertise-address={{ config['Genesis:ip'] }}
+        - --allow-privileged=true
+        - --anonymous-auth=false
+        - --bind-address=0.0.0.0
+        - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+        - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
+        - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
+        - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
+        - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
+        - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
+        - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
+        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+        - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
+        - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
+        - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
+        {%- for argument in config.get_path('Genesis:apiserver.arguments', []) %}
+        - "{{ argument }}"
+        {%- endfor %}

promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml

@@ -1,6 +0,0 @@
----
-kind: AdmissionConfiguration
-apiVersion: apiserver.k8s.io/v1alpha1
-plugins:
-- name: EventRateLimit
-  path: eventconfig.yaml

promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml

@@ -1,7 +0,0 @@
----
-kind: Configuration
-apiVersion: eventratelimit.admission.k8s.io/v1alpha1
-limits:
-- type: Server
-  qps: 100
-  burst: 1000

promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml

@@ -11,146 +11,130 @@ spec:
   dnsPolicy: Default
   hostNetwork: true
   containers:
-  - env:
+    - env:
-    - name: TILLER_NAMESPACE
+      - name: TILLER_NAMESPACE
-      value: kube-system
+        value: kube-system
-    image: {{ config['Genesis:images.helm.tiller'] }}
+      image: {{ config['Genesis:images.helm.tiller'] }}
-    command:
+      command:
-      - /tiller
+        - /tiller
-      - -logtostderr
+        - -logtostderr
-      - -v
+        - -v
-      - "99"
+        - "99"
-    imagePullPolicy: IfNotPresent
+      imagePullPolicy: IfNotPresent
-    livenessProbe:
+      livenessProbe:
-      failureThreshold: 3
+        failureThreshold: 3
-      httpGet:
+        httpGet:
-        path: /liveness
+          path: /liveness
-        port: 44135
+          port: 44135
-        scheme: HTTP
+          scheme: HTTP
-      initialDelaySeconds: 1
+        initialDelaySeconds: 1
-      periodSeconds: 10
+        periodSeconds: 10
-      successThreshold: 1
+        successThreshold: 1
-      timeoutSeconds: 1
+        timeoutSeconds: 1
-    name: tiller
-    ports:
-    - containerPort: 44134
       name: tiller
-      protocol: TCP
+      ports:
-    readinessProbe:
+      - containerPort: 44134
-      failureThreshold: 3
+        name: tiller
-      httpGet:
+        protocol: TCP
-        path: /readiness
+      readinessProbe:
-        port: 44135
+        failureThreshold: 3
-        scheme: HTTP
+        httpGet:
-      initialDelaySeconds: 1
+          path: /readiness
-      periodSeconds: 10
+          port: 44135
-      successThreshold: 1
+          scheme: HTTP
-      timeoutSeconds: 1
+        initialDelaySeconds: 1
-    resources: {}
+        periodSeconds: 10
-    terminationMessagePath: /dev/termination-log
+        successThreshold: 1
-    terminationMessagePolicy: File
+        timeoutSeconds: 1
-  - name: armada
+      resources: {}
-    image: {{ config['Genesis:images.armada'] }}
+      terminationMessagePath: /dev/termination-log
-    securityContext:
+      terminationMessagePolicy: File
-      runAsUser: 0
+    - name: armada
-    command:
+      image: {{ config['Genesis:images.armada'] }}
-      - /bin/bash
+      securityContext:
-      - -c
+        runAsUser: 0
-      - |-
+      command:
-        set -x
+        - /bin/bash
+        - -c
+        - |-
+          set -x
 
-        while true; do
+          while true; do
-            sleep 10
+              sleep 10
-            if armada \
+              if armada \
-                    apply \
+                      apply \
-                    --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
+                      --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
-                    --tiller-host 127.0.0.1 \
+                      --tiller-host 127.0.0.1 \
-                    /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
+                      /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
-                break
+                  break
-            fi
+              fi
-        done
+          done
+          touch /ipc/armada-done
+          sleep 10000
+      env:
+        - name: ARMADA_LOGFILE
+          value: /tmp/log/bootstrap-armada.log
+  {%- if config['KubernetesNetwork:proxy.url'] is defined %}
+        - name: HTTP_PROXY
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
+        - name: HTTPS_PROXY
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
+        - name: NO_PROXY
+          value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
+        - name: http_proxy
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
+        - name: https_proxy
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
+        - name: no_proxy
+          value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
+  {%- endif %}
+      volumeMounts:
+        - name: assets
+          mountPath: /etc/genesis/armada/assets
+        - name: auth
+          mountPath: /root/.kube
+        - name: ipc
+          mountPath: /ipc
+        - name: log
+          mountPath: /tmp/log
+    - name: monitor
+      image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
+      command:
+        - /bin/sh
+        - -c
+        - |-
+          set -x
 
-        touch /ipc/armada-done
+          while ! [ -e /ipc/armada-done ]; do
-        sleep 10000
+            sleep 5
-    env:
+          done
-      - name: ARMADA_LOGFILE
-        value: /tmp/log/bootstrap-armada.log
-{%- if config['KubernetesNetwork:proxy.url'] is defined %}
-      - name: HTTP_PROXY
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
-      - name: HTTPS_PROXY
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
-      - name: NO_PROXY
-        value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
-      - name: http_proxy
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
-      - name: https_proxy
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
-      - name: no_proxy
-        value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
-{%- endif %}
-    volumeMounts:
-      - name: assets
-        mountPath: /etc/genesis/armada/assets
-      - name: auth
-        mountPath: /root/.kube
-      - name: ipc
-        mountPath: /ipc
-      - name: log
-        mountPath: /tmp/log
-  - name: monitor
-    image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
-    command:
-      - /bin/sh
-      - -c
-      - |-
-        set -x
 
-        while ! [ -e /ipc/armada-done ]; do
+          rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml
-          sleep 5
+          sleep 10000
-        done
+      volumeMounts:
-
+        - name: ipc
-        rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml
+          mountPath: /ipc
-        sleep 10000
+        - name: manifest
-    volumeMounts:
+          mountPath: /etc/kubernetes/manifests
-      - name: ipc
+    - name: kubectl-apiserver
-        mountPath: /ipc
+      image: {{ config['Genesis:images.kubernetes.apiserver'] }}
-      - name: manifest
+      command:
-        mountPath: /etc/kubernetes/manifests
+        {%- for argument in config.bootstrap_apiserver_prefix() %}
-  - name: kubectl-apiserver
+        - "{{ argument }}"
-    image: {{ config['Genesis:images.kubernetes.apiserver'] }}
+        {%- endfor %}
-    command:
+{% include "genesis-apiserver.yaml" with context %}
-      {%- for argument in config.bootstrap_apiserver_prefix() %}
+        - --etcd-servers=https://localhost:12379
-      - "{{ argument }}"
+        - --insecure-port=8080
-      {%- endfor %}
+        - --secure-port=6444
-      - --advertise-address={{ config['Genesis:ip'] }}
+      env:
-      - --anonymous-auth=false
+        - name: KUBECONFIG
-      - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+          value: /etc/kubernetes/admin/config
-      - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+      volumeMounts:
-      - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem
+        - name: auth
-      - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem
+          mountPath: /etc/kubernetes/admin
-      - --insecure-port=8080
+        - name: config
-      - --secure-port=6444
+          mountPath: /etc/kubernetes/apiserver
-      - --bind-address=0.0.0.0
+          readOnly: true
-      - --allow-privileged=true
-      - --etcd-servers=https://localhost:12379
-      - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
-      - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
-      - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
-      - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-      - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
-      - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
-      - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
-      - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
-    env:
-      - name: KUBECONFIG
-        value: /etc/kubernetes/admin/config
-    volumeMounts:
-      - name: auth
-        mountPath: /etc/kubernetes/admin
-      - name: config
-        mountPath: /etc/kubernetes/apiserver
-        readOnly: true
   volumes:
     - name: assets
       hostPath:

promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml

@@ -19,25 +19,10 @@ spec:
         {%- for argument in config.bootstrap_apiserver_prefix() %}
         - "{{ argument }}"
         {%- endfor %}
-        - --advertise-address={{ config['Genesis:ip'] }}
+{% include "genesis-apiserver.yaml" with context %}
-        - --anonymous-auth=false
-        - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
-        - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
-        - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
-        - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
-        - --insecure-port=0
-        - --bind-address=0.0.0.0
-        - --secure-port=6443
-        - --allow-privileged=true
         - --etcd-servers=https://localhost:2379
-        - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
+        - --insecure-port=0
-        - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
+        - --secure-port=6443
-        - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
-        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-        - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
-        - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
-        - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
-        - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
       volumeMounts:
         - name: config
           mountPath: /etc/kubernetes/apiserver