Extend webhook-enabled apiserver chart

- Updates to the webhook-enabled apiserver chart to properly
  support certificate trust and allow for fragmented CAs for
  better security.

Change-Id: I56dee9d1ca4e0807d89ce6b0f3ab3fb5d4ea8c67

charts/apiserver-webhook/templates/bin/_webhook_start.sh.tpl

@@ -18,9 +18,20 @@ limitations under the License.
 
 set -xe
 
+SERVER_CERT_FILE=${SERVER_CERT_FILE:-"/etc/webhook_apiserver/pki/tls.crt"}
+SERVER_KEY_FILE=${SERVER_KEY_FILE:-"/etc/webhook_apiserver/pki/tls.key"}
+POLICY_FILE=${POLICY_FILE:-"/etc/webhook_apiserver/policy.json"}
+SERVER_PORT=${SERVER_PORT:-"8443"}
+KEYSTONE_CA_FILE=${KEYSTONE_CA_FILE:-"/etc/webhook_apiserver/pki/keystone.pem"}
+
 exec /bin/k8s-keystone-auth \
-  --tls-cert-file /opt/kubernetes-keystone-webhook/pki/tls.crt \
+  --v 5 \
-  --tls-private-key-file /opt/kubernetes-keystone-webhook/pki/tls.key \
+  --tls-cert-file "${SERVER_CERT_FILE}" \
-  --keystone-policy-file /etc/kubernetes-keystone-webhook/policy.json \
+  --tls-private-key-file "${SERVER_KEY_FILE}" \
-  --listen 127.0.0.1:8443 \
+  --keystone-policy-file "${POLICY_FILE}" \
+  --listen "127.0.0.1:${SERVER_PORT}" \
+{{- if hasKey .Values.certificates "keystone" }}
+  --keystone-ca-file "${KEYSTONE_CA_FILE}" \
+{{- end }}
   --keystone-url {{ tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+

charts/apiserver-webhook/templates/configmap-bin.yaml

@@ -16,13 +16,15 @@ limitations under the License.
 
 {{- if .Values.manifests.configmap_bin }}
 {{- $envAll := . }}
-
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ .Values.service.name }}-bin
+  name: {{ .Release.Name }}-bin
 data:
-  webhook_start.sh: |
+  ks-user.sh: |-
+{{- include "helm-toolkit.scripts.keystone_user" $envAll | indent 4 }}
+  webhook_start.sh: |-
 {{ tuple "bin/_webhook_start.sh.tpl" $envAll | include "helm-toolkit.utils.template" | indent 4 }}
+...
 {{- end }}

charts/apiserver-webhook/templates/configmap-certs.yaml

@@ -1,31 +0,0 @@
-{{/*
-Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if .Values.manifests.configmap_certs }}
-{{- $envAll := . }}
-
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ .Values.service.name }}-certs
-data:
-  cluster-ca.pem: {{ .Values.secrets.tls.ca | quote }}
-  apiserver.pem: {{ .Values.secrets.tls.cert | quote }}
-  etcd-client-ca.pem: {{ .Values.secrets.etcd.tls.ca | quote }}
-  etcd-client.pem: {{ .Values.secrets.etcd.tls.cert | quote }}
-  service-account.pub: {{ .Values.secrets.service_account.public_key | quote }}
-{{- end }}

charts/apiserver-webhook/templates/configmap-etc.yaml

@@ -21,8 +21,9 @@ limitations under the License.
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ .Values.service.name }}-etc
+  name: {{ .Release.Name }}-etc
 data:
+  service-account.pub: {{ .Values.secrets.service_account.public_key | quote }}
   webhook.kubeconfig: |
 {{ tuple "etc/_webhook.kubeconfig.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   policy.json: |

charts/apiserver-webhook/templates/deployment.yaml

@@ -14,13 +14,94 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{/*
+These local.* templates may be moved out of this chart into helm-toolkit
+in the future if there is desire to generalize this pattern. Otherwise
+in the future they will be moved into a separate helpers file.
+*/}}
+
+{{- define "local.tls_volume_name" -}}
+{{- $group := index . 0 -}}
+{{- $type := index . 1 -}}
+tls-{{ $group | replace "_" "-" }}-{{ $type | replace "_" "-" }}
+{{- end -}}
+
+{{- define "local.attach_all_bundles" }}
+{{- $envAll := . }}
+{{- range $group, $certs := $envAll.Values.certificates }}
+{{- range $type, $bundle := . }}
+{{ tuple $group $type $envAll | include "local.attach_cert_bundle" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- define "local.attach_cert_bundle" }}
+{{- $group := index . 0 }}
+{{- $type := index . 1 }}
+{{- $envAll := index . 2 }}
+- name: {{ tuple $group  $type | include "local.tls_volume_name" }}
+  secret:
+    secretName: {{ tuple $group $type $envAll | include "local.tls_secret_name" }}
+    defaultMode: 0444
+{{ end }}
+
+{{- define "local.mount_all_bundles" }}
+{{- $basepath := index . 0 }}
+{{- $envAll := index . 1 }}
+{{- range $group, $certs := $envAll.Values.certificates }}
+{{- range $type, $bundle := . }}
+{{ tuple $group $type $basepath $envAll | include "local.mount_cert_bundle" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- define "local.mount_cert_bundle" }}
+{{- $group := index . 0 }}
+{{- $type := index . 1 }}
+{{- $basepath := index . 2 }}
+{{- $envAll := index . 3 }}
+{{- $bundle := index $envAll.Values "certificates" $group $type }}
+{{- range tuple "ca" "cert" "key" }}
+{{- if hasKey $bundle . }}
+{{ tuple $group $type . $basepath $envAll | include "local.mount_cert_file" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- define "local.mount_cert_file" }}
+{{- $group := index . 0 }}
+{{- $type := index . 1 }}
+{{- $member := index . 2 }}
+{{- $basepath := index . 3 }}
+{{- $envAll := index . 4 }}
+- name: {{ tuple $group  $type | include "local.tls_volume_name" }}
+  mountPath: {{ tuple $group $type $basepath $member $envAll | include "local.cert_bundle_path" }}
+{{- if eq $member "ca" }}
+  subPath: ca.crt
+{{- else if eq $member "cert" }}
+  subPath: tls.crt
+{{- else if eq $member "key" }}
+  subPath: tls.key
+{{- end }}
+  readOnly: true
+{{- end }}
+
+{{- define "local.cert_bundle_path" -}}
+{{- $group := index . 0 -}}
+{{- $type := index . 1 -}}
+{{- $basepath := index . 2 -}}
+{{- $member := index . 3 -}}
+{{- $envAll := index . 4 -}}
+{{ $basepath }}/{{ $group }}-{{ $type }}-{{ $member }}.pem
+{{- end -}}
+
 {{- if .Values.manifests.deployment }}
 {{- $envAll := . }}
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: kubernetes-keystone-webhook
+  name: {{ .Release.Name }}-apiserver-webhook
   labels:
 {{ tuple $envAll "kubernetes-keystone-webhook" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 spec:
@@ -36,7 +117,7 @@ spec:
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
     spec:
-      dnsPolicy: ClusterFirstWithHostNet
+      dnsPolicy: ClusterFirst
       containers:
         - name: apiserver
           image: {{ .Values.images.tags.apiserver }}
@@ -50,93 +131,117 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
-
           command:
             {{- range .Values.command_prefix }}
             - {{ . }}
             {{- end }}
+            - --service-cluster-ip-range={{ $envAll.Values.network.service_cidr }}
             - --authorization-mode=Webhook
             - --advertise-address=$(POD_IP)
             - --anonymous-auth=false
             - --endpoint-reconciler-type=none
-            - --bind-address=0.0.0.0
+            - --bind-address=$(POD_IP)
-            - --secure-port={{ .Values.network.kubernetes_apiserver.port }}
+            - --secure-port={{ tuple "webhook_apiserver" "podport" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
             - --insecure-port=0
-            - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+            - --tls-cert-file={{ tuple "apiserver_webhook_pod" "server" $envAll.Values.conf.paths.pki "cert" $envAll | include "local.cert_bundle_path" }}
-            - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
+            - --tls-private-key-file={{ tuple "apiserver_webhook_pod" "server" $envAll.Values.conf.paths.pki "key" $envAll | include "local.cert_bundle_path" }}
-            - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
             - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-            - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+            - --kubelet-certificate-authority={{ tuple "kubelet" "server" $envAll.Values.conf.paths.pki "ca" $envAll | include "local.cert_bundle_path" }}
-            - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem
+            - --kubelet-client-certificate={{ tuple "kubelet" "client" $envAll.Values.conf.paths.pki "cert" $envAll | include "local.cert_bundle_path" }}
-            - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem
+            - --kubelet-client-key={{ tuple "kubelet" "client" $envAll.Values.conf.paths.pki "key" $envAll | include "local.cert_bundle_path" }}
             - --etcd-servers={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
-            - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
+            - --etcd-cafile={{ tuple "etcd" "server" $envAll.Values.conf.paths.pki "ca" $envAll | include "local.cert_bundle_path" }}
-            - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
+            - --etcd-certfile={{ tuple "etcd" "client" $envAll.Values.conf.paths.pki "cert" $envAll | include "local.cert_bundle_path" }}
-            - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
+            - --etcd-keyfile={{ tuple "etcd" "client" $envAll.Values.conf.paths.pki "key" $envAll | include "local.cert_bundle_path" }}
             - --allow-privileged=true
-            - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
+            - --service-account-key-file={{ $envAll.Values.conf.paths.sapubkey }}
-            - --authentication-token-webhook-config-file=/etc/kubernetes/apiserver/webhook.kubeconfig
+            - --authentication-token-webhook-config-file={{ $envAll.Values.conf.paths.conf }}
-            - --authorization-webhook-config-file=/etc/kubernetes/apiserver/webhook.kubeconfig
+            - --authorization-webhook-config-file={{ $envAll.Values.conf.paths.conf }}
-          ports:
-            - containerPort: {{ .Values.network.kubernetes_apiserver.port }}
           readinessProbe:
             tcpSocket:
-              port: 6443
+              port: {{ tuple "webhook_apiserver" "podport" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-            initialDelaySeconds: 5
+{{ $envAll.Values.pod.probes.readinessProbe | toYaml | indent 12 }}
-            periodSeconds: 10
           livenessProbe:
             tcpSocket:
-              port: 6443
+              port: {{ tuple "webhook_apiserver" "podport" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-            failureThreshold: 3
+{{ $envAll.Values.pod.probes.livenessProbe | toYaml | indent 12 }}
-            initialDelaySeconds: 15
-            periodSeconds: 20
           volumeMounts:
-            - name: etc
+            - name: etc-apiserver
-              mountPath: /etc/kubernetes/apiserver
+              mountPath: {{ $envAll.Values.conf.paths.base }}
-            - name: {{ .Values.service.name }}-etc
+            - name: etc-apiserver-pki
-              mountPath: /etc/kubernetes/apiserver/webhook.kubeconfig
+              mountPath: {{ $envAll.Values.conf.paths.pki }}
+            - name: configmap-etc
+              mountPath: {{ $envAll.Values.conf.paths.sapubkey }}
+              subPath: service-account.pub
+              readOnly: true
+            - name: configmap-etc
+              mountPath: {{ $envAll.Values.conf.paths.conf }}
               subPath: webhook.kubeconfig
               readOnly: true
-        - name: kubernetes-keystone-webhook
+{{ tuple "keystone_webhook" "server" "ca" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_file" | indent 12 }}
+{{ tuple "apiserver_webhook_pod" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
+{{ tuple "kubelet" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
+{{ tuple "kubelet" "client" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
+{{ tuple "etcd" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
+{{ tuple "etcd" "client" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
+        - name: webhook
 {{ tuple $envAll "kubernetes_keystone_webhook" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           command:
             - /tmp/webhook_start.sh
+          env:
+{{- with $env := dict "ksUserSecret" .Values.secrets.identity.webhook }}
+{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
+{{- end }}
+            - name: SERVER_CERT_FILE
+              value: {{ tuple "keystone_webhook" "server" $envAll.Values.conf.paths.pki "cert" $envAll | include "local.cert_bundle_path" | quote }}
+            - name: SERVER_KEY_FILE
+              value: {{ tuple "keystone_webhook" "server" $envAll.Values.conf.paths.pki "key" $envAll | include "local.cert_bundle_path" | quote }}
+            - name: POLICY_FILE
+              value: {{ $envAll.Values.conf.paths.policy | quote }}
+            - name: SERVER_PORT
+              value: {{ tuple "webhook_apiserver" "podport" "webhook" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+{{- if hasKey .Values.certificates "keystone" }}
+            - name: KEYSTONE_CA_FILE
+              value: {{ tuple "keystone" "server" $envAll.Values.conf.paths.pki "ca" $envAll | include "local.cert_bundle_path" | quote }}
+{{- end }}
           volumeMounts:
-            - name: etc-kubernetes-keystone-webhook
+            - name: etc-webhook
-              mountPath: /etc/kubernetes-keystone-webhook
+              mountPath: {{ $envAll.Values.conf.paths.base }}
-            - name: key-kubernetes-keystone-webhook
+            - name: etc-webhook-pki
-              mountPath: /opt/kubernetes-keystone-webhook/pki/tls.crt
+              mountPath: {{ $envAll.Values.conf.paths.pki }}
-              subPath: tls.crt
+            - name: configmap-etc
-              readOnly: true
+              mountPath: {{ $envAll.Values.conf.paths.policy }}
-            - name: key-kubernetes-keystone-webhook
-              mountPath: /opt/kubernetes-keystone-webhook/pki/tls.key
-              subPath: tls.key
-              readOnly: true
-            - name: {{ .Values.service.name }}-etc
-              mountPath: /etc/kubernetes-keystone-webhook/policy.json
               subPath: policy.json
               readOnly: true
-            - name: {{ .Values.service.name }}-bin
+            - name: configmap-bin
               mountPath: /tmp/webhook_start.sh
               subPath: webhook_start.sh
               readOnly: true
+{{ tuple "keystone_webhook" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
       volumes:
-        - name: etc
+{{- if hasKey .Values.certificates "keystone" }}
-          hostPath:
+{{ tuple "keystone" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
-            path: {{ .Values.apiserver.host_etc_path }}
+{{- end }}
-        - name: etc-kubernetes-keystone-webhook
+{{ include "local.attach_all_bundles" $envAll | indent 8 }}
-          emptyDir: {}
+        - name: etc-apiserver
-        - name: key-kubernetes-keystone-webhook
+          emptyDir: {}
-          secret:
+        - name: etc-apiserver-pki
-            secretName: {{ $envAll.Values.secrets.certificates.api }}
+          emptyDir: {}
-            defaultMode: 0444
+        - name: etc-webhook
-        - name: {{ .Values.service.name }}-etc
+          emptyDir: {}
-          configMap:
+        - name: etc-webhook-pki
-            name: {{ .Values.service.name }}-etc
+          emptyDir: {}
-            defaultMode: 0444
+        - name: configmap-etc
-        - name: {{ .Values.service.name }}-bin
+          configMap:
-          configMap:
+            name: {{ .Release.Name }}-etc
-            name: {{ .Values.service.name }}-bin
+            defaultMode: 0444
-            defaultMode: 0555
+        - name: configmap-bin
+          configMap:
+            name: {{ .Release.Name }}-bin
+            defaultMode: 0555
+        - name: tls-apiserver-webhook-public-server
+          secret:
+            defaultMode: 292
+            secretName: {{ .Values.secrets.tls.webhook_apiserver.api.public }}
 {{- end }}

charts/apiserver-webhook/templates/etc/_kubeconfig.yaml.tpl

@@ -1,34 +0,0 @@
-# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
----
-apiVersion: v1
-clusters:
-- cluster:
-    server: https://127.0.0.1:{{ .Values.network.kubernetes_apiserver.port }}
-    certificate-authority: pki/cluster-ca.pem
-  name: kubernetes
-contexts:
-- context:
-    cluster: kubernetes
-    user: apiserver
-  name: apiserver@kubernetes
-current-context: apiserver@kubernetes
-kind: Config
-preferences: {}
-users:
-- name: apiserver
-  user:
-    client-certificate: pki/apiserver.pem
-    client-key: pki/apiserver-key.pem

charts/apiserver-webhook/templates/etc/_webhook.kubeconfig.tpl

@@ -2,7 +2,8 @@ apiVersion: v1
 clusters:
   - cluster:
       insecure-skip-tls-verify: false
-      server: https://127.0.0.1:8443/webhook
+      server: https://127.0.0.1:{{ tuple "webhook_apiserver" "podport" "webhook" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/webhook
+      certificate-authority: {{ tuple "keystone_webhook" "server" .Values.conf.paths.pki "ca" . | include "local.cert_bundle_path" | quote }}
     name: webhook
 contexts:
   - context:

charts/apiserver-webhook/templates/ingress-api.yaml

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and .Values.manifests.ingress_api .Values.network.kubernetes_apiserver.ingress.public }}
+{{- if and .Values.manifests.ingress_api .Values.network.api.ingress.public }}
-{{- $ingressOpts := dict "envAll" . "backendService" "kubernetes_apiserver" "backendServiceType" "kubernetes_apiserver" "backendPort" "https" -}}
+{{- $ingressOpts := dict "envAll" . "backendServiceType" "webhook_apiserver" "backendPort" "https" -}}
-{{- $ingressOpts | include "helm-toolkit.manifests.ingress" -}}
+{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
 {{- end }}

charts/apiserver-webhook/templates/job-ks-user.yaml

@@ -1,5 +1,5 @@
 {{/*
-Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,15 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests.secret }}
+{{- if .Values.manifests.job_ks_user }}
-{{- $envAll := . }}
+{{ $cm_name := printf "%s-bin" .Release.Name }}
----
+{{- $ksUserJob := dict "envAll" . "serviceName" "webhook_apiserver" "configMapBin" $cm_name "serviceUser" "webhook" -}}
-apiVersion: v1
+{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }}
-kind: Secret
-metadata:
-  name: {{ .Values.service.name }}-keys
-type: Opaque
-data:
-  apiserver-key.pem: {{ .Values.secrets.tls.key | b64enc }}
-  etcd-client-key.pem: {{ .Values.secrets.etcd.tls.key | b64enc }}
 {{- end }}

charts/apiserver-webhook/templates/secret-ingress-tls.yaml

@@ -1,19 +0,0 @@
-{{/*
-Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if .Values.manifests.secret_ingress_tls }}
-{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendService" "kubernetes_apiserver" "backendServiceType" "kubernetes_apiserver" ) }}
-{{- end }}

charts/apiserver-webhook/templates/secret-keystone.yaml

@@ -16,7 +16,7 @@ limitations under the License.
 
 {{- if .Values.manifests.secret_keystone }}
 {{- $envAll := . }}
-{{- range $key1, $userClass := tuple "admin" }}
+{{- range $key1, $userClass := tuple "admin" "webhook" }}
 {{- $secretName := index $envAll.Values.secrets.identity $userClass }}
 ---
 apiVersion: v1

charts/apiserver-webhook/templates/secret-tls.yaml

@@ -0,0 +1,73 @@
+{{/*
+Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "local.tls_secret_name" -}}
+{{- $group := index . 0 -}}
+{{- $type := index . 1 -}}
+{{- $envAll := index . 2 -}}
+{{ printf "%s-%s-%s" $envAll.Release.Name $group $type | replace "_" "-" }}
+{{- end -}}
+
+{{- define "local.tls_secret" }}
+{{- $group := index . 0 }}
+{{- $type := index . 1 }}
+{{- $bundle := index . 2 }}
+{{- $envAll := index . 3 }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ tuple $group $type $envAll | include "local.tls_secret_name" }}
+  namespace: {{ $envAll.Release.Namespace }}
+type: opaque
+data:
+  {{- if hasKey $bundle "ca" }}
+  ca.crt: |-
+{{ $bundle.ca | b64enc | indent 4 }}
+  {{- end }}
+  {{- if hasKey $bundle "cert" }}
+  tls.crt: |-
+{{ $bundle.cert | b64enc | indent 4 }}
+  {{- end }}
+  {{- if hasKey $bundle "key" }}
+  tls.key: |-
+{{ $bundle.key | b64enc | indent 4 }}
+  {{- end }}
+...
+{{- end -}}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.secrets.tls.webhook_apiserver.api.public }}
+  namespace: {{ .Release.Namespace }}
+type: opaque
+data:
+  ca.crt: |-
+{{ .Values.secrets.tls.webhook_apiserver.api.server.ca | b64enc | indent 4 }}
+  tls.crt: |-
+{{ .Values.secrets.tls.webhook_apiserver.api.server.cert | b64enc | indent 4 }}
+  tls.key: |-
+{{ .Values.secrets.tls.webhook_apiserver.api.server.key | b64enc | indent 4 }}
+...
+{{- if .Values.manifests.secret_tls }}
+{{- $envAll := . }}
+{{- range $group, $certs := .Values.certificates }}
+{{- range $type, $bundle := $certs }}
+{{ tuple $group $type $bundle $envAll | include "local.tls_secret" }}
+{{- end }}
+{{- end }}
+{{- end }}

charts/apiserver-webhook/templates/secret-webhook.yaml

@@ -1,28 +0,0 @@
-{{/*
-Copyright 2018 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if .Values.manifests.secret_webhook }}
-{{- $envAll := . }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ $envAll.Values.secrets.certificates.api }}
-type: kubernetes.io/tls
-data:
-  tls.crt: {{ $envAll.Values.endpoints.kubernetes.auth.api.tls.crt | default "" | b64enc }}
-  tls.key: {{ $envAll.Values.endpoints.kubernetes.auth.api.tls.key | default "" | b64enc }}
-{{- end }}

charts/apiserver-webhook/templates/service-ingress-api.yaml

@@ -1,6 +1,5 @@
 {{/*
 Copyright 2017 The Openstack-Helm Authors.
-Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,7 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and .Values.manifests.service_ingress .Values.network.kubernetes_apiserver.ingress.public }}
+{{- if and .Values.manifests.ingress_api .Values.network.api.ingress.public }}
-{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "kubernetes-keystone-webhook" -}}
+{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "webhook_apiserver" -}}
 {{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }}
 {{- end }}
+

charts/apiserver-webhook/templates/service.yaml

@@ -20,15 +20,15 @@ limitations under the License.
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ .Values.service.name }}
+  name: {{ tuple "webhook_apiserver" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
   annotations:
     service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
 spec:
   ports:
     - name: https
-      port: {{ .Values.network.kubernetes_apiserver.port }}
+      port: {{ tuple "webhook_apiserver" "default" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
       protocol: TCP
-      targetPort: {{ .Values.network.kubernetes_apiserver.port }}
+      targetPort: {{ tuple "webhook_apiserver" "podport" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
   selector:
 {{ tuple $envAll "kubernetes-keystone-webhook" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 {{- end }}

charts/apiserver-webhook/values.yaml

@@ -21,6 +21,7 @@ images:
     scripted_test: docker.io/openstackhelm/heat:newton
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
     image_repo_sync: docker.io/docker:17.07.0
+    ks_user: docker.io/openstackhelm/heat:ocata
   pull_policy: IfNotPresent
   local_registry:
     active: false
@@ -30,80 +31,101 @@ images:
 
 labels:
   kubernetes_apiserver:
-    node_selector_key: kubernetes-apiserver
+    node_selector_key: apiserver-webhook
+    node_selector_value: enabled
+  job:
+    node_selector_key: apiserver-webhook
     node_selector_value: enabled
 
 command_prefix:
   - /apiserver
   - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
-  - --service-cluster-ip-range=10.96.0.0/16
   - --v=5
 
-apiserver:
-  host_etc_path: /etc/kubernetes/apiserver
-
 network:
-  kubernetes_apiserver:
+  pod_cidr: '10.97.0.0/16'
+  service_cidr: '10.96.0.0/16'
+  api:
     ingress:
       public: true
       classes:
-        namespace: "nginx-cluster"
+        namespace: "nginx"
         cluster: "nginx-cluster"
       annotations:
         nginx.ingress.kubernetes.io/rewrite-target: /
         nginx.ingress.kubernetes.io/proxy-read-timeout: "120"
         nginx.ingress.kubernetes.io/ssl-redirect: "true"
         nginx.ingress.kubernetes.io/secure-backends: "true"
-    name: kubernetes-apiserver
+    name: webhook_apiserver
-    port: 6443
+#
-    node_port:
+# Insert TLS certificates, keys and CAs
-      enabled: false
+# here. Server is for server-terminated TLS (basic)
-      port: 31943
+# and client is for mTLS. Each group of certificates
-
+# will generate two secrets <groupname>-client and <groupname>-server
-service:
+# built to the kubernetes.io/tls secret type with keys 'tls.crt', 'tls.key'
-  name: kubernetes-webhook-apiserver
+# and 'ca.crt'
-  ip: null
+#
-
+certificates:
-secrets:
+  apiserver_webhook_pod:
-  tls:
+    server:
-    ca: placeholder
-    cert: placeholder
-    key: placeholder
-  service_account:
-    public_key: placeholder
-  etcd:
-    tls:
-      ca: placeholder
       cert: placeholder
       key: placeholder
-  identity:
+      ca: placeholder
-    admin: kubernetes-keystone-webhook-admin
+  keystone_webhook:
-  certificates:
+    server:
-    api: kubernetes-keystone-webhook-certs
+      cert: placeholder
+      key: placeholder
+      ca: placeholder
+  kubelet:
+    client:
+      cert: placeholder
+      key: placeholder
+    server:
+      ca: placeholder
+  etcd:
+    client:
+      cert: placeholder
+      key: placeholder
+    server:
+      ca: placeholder
 
-kubernetes_keystone_webhook:
+secrets:
-  port: 8443
+  service_account:
-  endpoints: https://k8sksauth-api.kube-system.svc.cluster.local
+    public_key: placeholder
+  identity:
+    admin: apiserver-webhook-keystone-creds-admin
+    webhook: apiserver-webhook-keystone-creds-webhook
+  tls:
+    webhook_apiserver:
+      api:
+        public: apiserver-webhook-public
+        server:
+          cert: placeholder
+          key: placeholder
+          ca: placeholder
 
 # typically overriden by environmental
 # values, but should include all endpoints
 # required by this chart
 endpoints:
   cluster_domain_suffix: cluster.local
-  kubernetes_apiserver:
+  webhook_apiserver:
-    name: kubernetes-webhook-apiserver
+    name: webhook_apiserver
     hosts:
-      default: keystone
+      default: apiserver-webhook
-      internal: keystone-api
+      internal: apiserver-webhook-int
     port:
-      https:
+      api:
         default: 6443
         public: 443
+      webhook:
+        podport: 8443
     path:
       default: /
+      webhook: /webhook
     scheme:
-      default: http
+      default: https
-      public: http
+      public: https
     host_fqdn_override:
       default: null
       # NOTE: this chart supports TLS for fqdn over-ridden public
@@ -113,12 +135,6 @@ endpoints:
       #   tls:
       #     crt: null
       #     key: null
-  kubernetes:
-    auth:
-      api:
-        tls:
-          crt: null
-          key: null
   identity:
     name: keystone
     namespace: null
@@ -130,6 +146,14 @@ endpoints:
         project_name: admin
         user_domain_name: default
         project_domain_name: default
+      webhook:
+        region_name: RegionOne
+        username: webhook
+        password: password
+        project_name: service
+        user_domain_name: default
+        project_domain_name: default
+        role: admin
     hosts:
       default: keystone
       internal: keystone-api
@@ -143,22 +167,6 @@ endpoints:
       api:
         default: 80
         internal: 5000
-  kubernetes_keystone_webhook:
-    namespace: null
-    name: k8sksauth
-    hosts:
-      default: k8sksauth-api
-      public: k8sksauth
-    host_fqdn_override:
-      default: null
-    path:
-      default: /webhook
-    scheme:
-      default: https
-    port:
-      api:
-        default: 8443
-        public: 443
   etcd:
     name: etcd
     namespace: kube-system
@@ -182,6 +190,14 @@ pod:
   replicas:
     apiserver: 1
     api: 1
+  probes:
+    readinessProbe:
+      initialDelaySeconds: 5
+      periodSeconds: 10
+    livenessProbe:
+      failureThreshold: 3
+      initialDelaySeconds: 15
+      periodSeconds: 20
   lifecycle:
     upgrades:
       daemonsets:
@@ -232,6 +248,12 @@ pod:
       init_container: null
       kubernetes_keystone_webhook_tests: null
 conf:
+  paths:
+    base: '/etc/webhook_apiserver/'
+    pki: '/etc/webhook_apiserver/pki'
+    conf: '/etc/webhook_apiserver/webhook.kubeconfig'
+    policy: '/etc/webhook_apiserver/conf/policy.json'
+    sapubkey: '/etc/webhook_apiserver/pki/service-accounts.pub'
   policy:
     - resource:
         verbs:
@@ -273,23 +295,35 @@ conf:
           - "*"
         resources:
           - "*"
-        namespace: "openstack"
+        namespace: "ucp"
         version: "*"
       match:
         - type: project
           values:
-            - openstack-system
+            - ucp-admin
+            - airship-admin
+
+dependencies:
+  static:
+    ks_user:
+      services:
+        - service: identity
+          endpoint: internal
+    api:
+      jobs:
+        - webhook-apiserver-ks-user
+      services:
+        - service: identity
+          endpoint: internal
 
 manifests:
   configmap_bin: true
   configmap_certs: true
   configmap_etc: true
+  job_ks_user: true
   deployment: true
-  ingress_api: false
+  ingress_api: true
   pod_test: false
-  kubernetes_apiserver: true
+  secret_keystone: true
-  secret: true
+  secret_tls: true
-  secret_ingress_tls: false
-  secret_webhook: true
   service: true
-  service_ingress: false