From 13997310961ac0473916322c29e1030b04181a00 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Mon, 27 Aug 2018 16:12:38 -0500 Subject: [PATCH] Use separate CA for kubelet authorization This increases isolation of actions against the node API. With the previous combined CA approach, each node would have a valid key to talk to each other node. With this separated approach, only the API servers will have keys with access to the node APIs. Change-Id: I2705016eb963ca9d2cc2a344047677f4b2cc3025 --- .../apiserver/templates/configmap-certs.yaml | 2 ++ .../etc/_kubernetes-apiserver.yaml.tpl | 4 +-- .../apiserver/templates/secret-apiserver.yaml | 1 + charts/apiserver/values.yaml | 12 +++++++ examples/basic/PKICatalog.yaml | 5 +++ examples/basic/armada-resources.yaml | 36 ++++++++++++------- examples/complete/PKICatalog.yaml | 5 +++ .../etc/kubernetes/pki/kubelet-client-ca.pem | 1 + .../common/etc/systemd/system/kubelet.service | 2 +- .../apiserver/pki/kubelet-client-ca.pem | 1 + .../apiserver/pki/kubelet-client-key.pem | 1 + .../genesis/apiserver/pki/kubelet-client.pem | 1 + .../manifests/kubernetes-apiserver.yaml | 6 ++-- 13 files changed, 58 insertions(+), 19 deletions(-) create mode 100644 promenade/templates/roles/common/etc/kubernetes/pki/kubelet-client-ca.pem create mode 100644 promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client-ca.pem create mode 100644 promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client-key.pem create mode 100644 promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client.pem diff --git a/charts/apiserver/templates/configmap-certs.yaml b/charts/apiserver/templates/configmap-certs.yaml index 34d412e0..3e528c76 100644 --- a/charts/apiserver/templates/configmap-certs.yaml +++ b/charts/apiserver/templates/configmap-certs.yaml @@ -28,4 +28,6 @@ data: etcd-client-ca.pem: {{ .Values.secrets.etcd.tls.ca | quote }} etcd-client.pem: {{ .Values.secrets.etcd.tls.cert | quote }} service-account.pub: {{ .Values.secrets.service_account.public_key | quote }} + kubelet-client-ca.pem: {{ .Values.secrets.kubelet.tls.ca | default .Values.secrets.tls.ca | quote }} + kubelet-client.pem: {{ .Values.secrets.kubelet.tls.cert | default .Values.secrets.tls.cert | quote }} {{- end }} diff --git a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl index 7919503a..0d2f36da 100644 --- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl +++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl @@ -54,8 +54,8 @@ spec: - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem - - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem - - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem + - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem + - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem - --etcd-servers={{ .Values.apiserver.etcd.endpoints }} - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem diff --git a/charts/apiserver/templates/secret-apiserver.yaml b/charts/apiserver/templates/secret-apiserver.yaml index f32f6bfd..ed168d16 100644 --- a/charts/apiserver/templates/secret-apiserver.yaml +++ b/charts/apiserver/templates/secret-apiserver.yaml @@ -25,4 +25,5 @@ type: Opaque data: apiserver-key.pem: {{ .Values.secrets.tls.key | b64enc }} etcd-client-key.pem: {{ .Values.secrets.etcd.tls.key | b64enc }} + kubelet-client-key.pem: {{ .Values.secrets.kubelet.tls.key | default .Values.secrets.tls.key | b64enc }} {{- end }} diff --git a/charts/apiserver/values.yaml b/charts/apiserver/values.yaml index 7552e97c..f4bc653f 100644 --- a/charts/apiserver/values.yaml +++ b/charts/apiserver/values.yaml @@ -33,6 +33,10 @@ anchor: files_to_copy: - source: /certs/apiserver.pem dest: /etc/kubernetes/apiserver/pki/apiserver.pem + - source: /certs/kubelet-client.pem + dest: /etc/kubernetes/apiserver/pki/kubelet-client.pem + - source: /certs/kubelet-client-ca.pem + dest: /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem - source: /certs/cluster-ca.pem dest: /etc/kubernetes/apiserver/pki/cluster-ca.pem - source: /certs/etcd-client-ca.pem @@ -43,6 +47,8 @@ anchor: dest: /etc/kubernetes/apiserver/pki/service-account.pub - source: /keys/apiserver-key.pem dest: /etc/kubernetes/apiserver/pki/apiserver-key.pem + - source: /keys/kubelet-client-key.pem + dest: /etc/kubernetes/apiserver/pki/kubelet-client-key.pem - source: /keys/etcd-client-key.pem dest: /etc/kubernetes/apiserver/pki/etcd-client-key.pem - source: /tmp/etc/kubernetes-apiserver.yaml @@ -97,6 +103,12 @@ secrets: ca: placeholder cert: placeholder key: placeholder + kubelet: + tls: + ca: null + cert: null + key: null + # typically overriden by environmental # values, but should include all endpoints diff --git a/examples/basic/PKICatalog.yaml b/examples/basic/PKICatalog.yaml index b1d0a134..fda5234c 100644 --- a/examples/basic/PKICatalog.yaml +++ b/examples/basic/PKICatalog.yaml @@ -63,6 +63,11 @@ data: common_name: armada groups: - system:masters + kubelet: + description: CA for Kubernetes node interactions + certificates: + - document_name: apiserver-kubelet-client + common_name: apiserver-kubelet-client kubernetes-etcd: description: Certificates for Kubernetes's etcd servers certificates: diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index 904c17d8..78782acd 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -664,7 +664,6 @@ metadata: path: . dest: path: .values.secrets.tls.ca - - src: schema: deckhand/Certificate/v1 @@ -679,6 +678,29 @@ metadata: path: . dest: path: .values.secrets.tls.key + + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubelet + path: . + dest: + path: .values.secrets.kubelet.tls.ca + - + src: + schema: deckhand/Certificate/v1 + name: apiserver-kubelet-client + path: . + dest: + path: .values.secrets.kubelet.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: apiserver-kubelet-client + path: . + dest: + path: .values.secrets.kubelet.tls.key + - src: schema: deckhand/CertificateAuthority/v1 @@ -731,18 +753,6 @@ data: tags: anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.2 apiserver: gcr.io/google_containers/hyperkube-amd64:v1.10.2 - secrets: - service_account: - public_key: placeholder - tls: - ca: placeholder - cert: placeholder - key: placeholder - etcd: - tls: - ca: placeholder - cert: placeholder - key: placeholder network: kubernetes_service_ip: 10.96.0.1 pod_cidr: 10.97.0.0/16 diff --git a/examples/complete/PKICatalog.yaml b/examples/complete/PKICatalog.yaml index 07f98f59..41a0802a 100644 --- a/examples/complete/PKICatalog.yaml +++ b/examples/complete/PKICatalog.yaml @@ -70,6 +70,11 @@ data: common_name: armada groups: - system:masters + kubelet: + description: CA for Kubernetes node interactions + certificates: + - document_name: apiserver-kubelet-client + common_name: apiserver-kubelet-client kubernetes-etcd: description: Certificates for Kubernetes's etcd servers certificates: diff --git a/promenade/templates/roles/common/etc/kubernetes/pki/kubelet-client-ca.pem b/promenade/templates/roles/common/etc/kubernetes/pki/kubelet-client-ca.pem new file mode 100644 index 00000000..88d8720a --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/pki/kubelet-client-ca.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubelet', default=config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes')) }} diff --git a/promenade/templates/roles/common/etc/systemd/system/kubelet.service b/promenade/templates/roles/common/etc/systemd/system/kubelet.service index b3c3ef65..cb7a7a36 100644 --- a/promenade/templates/roles/common/etc/systemd/system/kubelet.service +++ b/promenade/templates/roles/common/etc/systemd/system/kubelet.service @@ -7,7 +7,7 @@ After=network-online.target ExecStart=/opt/kubernetes/bin/kubelet \ --allow-privileged=true \ --anonymous-auth=false \ - --client-ca-file=/etc/kubernetes/pki/cluster-ca.pem \ + --client-ca-file=/etc/kubernetes/pki/kubelet-client-ca.pem \ --cluster-dns={{ config['KubernetesNetwork:dns.service_ip'] }} \ --cluster-domain={{ config['KubernetesNetwork:dns.cluster_domain'] }} \ --hostname-override={{ config.get_first('Genesis:hostname', 'KubernetesNode:hostname') }} \ diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client-ca.pem b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client-ca.pem new file mode 100644 index 00000000..88d8720a --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client-ca.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubelet', default=config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes')) }} diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client-key.pem b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client-key.pem new file mode 100644 index 00000000..83bbc378 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client-key.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateKey/v1', name='apiserver-kubelet-client', default=config.get(schema='deckhand/CertificateKey/v1', name='apiserver')) }} diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client.pem b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client.pem new file mode 100644 index 00000000..8252687a --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/kubelet-client.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/Certificate/v1', name='apiserver-kubelet-client', default=config.get(schema='deckhand/Certificate/v1', name='apiserver')) }} diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml index 9a98509f..ef27e8b5 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml @@ -24,9 +24,9 @@ spec: - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds - --anonymous-auth=false - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem - - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem - - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem - - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem + - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem + - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem + - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem - --insecure-port=0 - --bind-address=0.0.0.0 - --secure-port=6443