Browse Source

k8s_fedora: Add kubelet authentication/authorization

* disable kubelet anonymous-auth
* enable kubelet webhook-(token) authorization
* disable kubelet cadvisor and read-only ports
* listen kubelet only on internal ipv4 ip
* update kubelet certs
* Update heapster RBAC to access kubelets
* update api config to access kubelet over https

Closes-Bug: #1758672
Change-Id: I2c6046ce5921a63a2d56f51435433497b1ff30ba
changes/13/556213/2
Spyros Trigazis 4 years ago
parent
commit
205e8adafa
  1. 3
      magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
  2. 6
      magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
  3. 36
      magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh
  4. 8
      magnum/drivers/common/templates/kubernetes/fragments/make-cert-client.sh
  5. 12
      releasenotes/notes/k8s_fedora_protect_kubelet-8468ddcb92c2a624.yaml

3
magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@ -28,6 +28,9 @@ else
KUBE_API_ARGS="$KUBE_API_ARGS --authorization-mode=Node,RBAC --tls-cert-file=$CERT_DIR/server.crt"
KUBE_API_ARGS="$KUBE_API_ARGS --tls-private-key-file=$CERT_DIR/server.key"
KUBE_API_ARGS="$KUBE_API_ARGS --client-ca-file=$CERT_DIR/ca.crt"
KUBE_API_ARGS="$KUBE_API_ARGS --tls-ca-file=${CERT_DIR}/ca.crt"
KUBE_API_ARGS="$KUBE_API_ARGS --service-account-key-file=${CERT_DIR}/server.key"
KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-certificate-authority=${CERT_DIR}/ca.crt --kubelet-client-certificate=${CERT_DIR}/server.crt --kubelet-client-key=${CERT_DIR}/server.key --kubelet-https=true"
fi
KUBE_ADMISSION_CONTROL=""

6
magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh

@ -29,6 +29,9 @@ fi
KUBE_MASTER_URI="$KUBE_PROTOCOL://$KUBE_MASTER_IP:$KUBE_API_PORT"
if [ -z "${KUBE_NODE_IP}" ]; then
KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
fi
HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
cat << EOF >> ${KUBELET_KUBECONFIG}
apiVersion: v1
@ -98,7 +101,8 @@ sed -i '
# the option --hostname-override for kubelet uses the hostname to register the node.
# Using any other name will break the load balancer and cinder volume features.
mkdir -p /etc/kubernetes/manifests
KUBELET_ARGS="--pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=4194 --kubeconfig ${KUBELET_KUBECONFIG} --hostname-override=${HOSTNAME_OVERRIDE}"
KUBELET_ARGS="--pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --kubeconfig ${KUBELET_KUBECONFIG} --hostname-override=${HOSTNAME_OVERRIDE}"
KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}"

36
magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh

@ -395,7 +395,7 @@ spec:
imagePullPolicy: IfNotPresent
command:
- /heapster
- --source=kubernetes:https://kubernetes.default
- --source=kubernetes:https://kubernetes.default?insecure=false&useServiceAccount=true&kubeletPort=10250&kubeletHttps=true
${INFLUX_SINK}
---
apiVersion: v1
@ -425,6 +425,40 @@ roleRef:
kind: ClusterRole
name: system:heapster
subjects:
- kind: ServiceAccount
name: heapster
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:heapster-to-kubelet
rules:
- apiGroups:
- ""
resources:
- nodes/proxy
- nodes/stats
- nodes/log
- nodes/spec
- nodes/metrics
verbs:
- "*"
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: system:heapter-kubelet
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:heapster-to-kubelet
subjects:
- kind: ServiceAccount
name: heapster
namespace: kube-system

8
magnum/drivers/common/templates/kubernetes/fragments/make-cert-client.sh

@ -30,6 +30,10 @@ else
VERIFY_CA="-k"
fi
if [ -z "${KUBE_NODE_IP}" ]; then
KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
fi
cert_dir=/etc/kubernetes/certs
mkdir -p "$cert_dir"
@ -93,6 +97,7 @@ EOF
#Kubelet Certs
INSTANCE_NAME=$(hostname --short | sed 's/\.novalocal//')
HOSTNAME=$(hostname)
cat > ${cert_dir}/kubelet.conf <<EOF
[req]
@ -107,8 +112,9 @@ C=US
ST=TX
L=Austin
[req_ext]
subjectAltName = IP:${KUBE_NODE_IP},DNS:${INSTANCE_NAME},DNS:${HOSTNAME}
keyUsage=critical,digitalSignature,keyEncipherment
extendedKeyUsage=clientAuth
extendedKeyUsage=clientAuth,serverAuth
EOF
#kube-proxy Certs

12
releasenotes/notes/k8s_fedora_protect_kubelet-8468ddcb92c2a624.yaml

@ -0,0 +1,12 @@
---
fixes:
- |
Fix bug #1758672 [1] to protect kubelet in the k8s_fedora_atomic driver.
Before this patch kubelet was listening to 0.0.0.0 and for clusters with
floating IPs the kubelet was exposed. Also, even on clusters without fips
the kubelet was exposed inside the cluster. This patch allows access to
the kubelet only over https and with the appropriate roles. The apiserver
and heapster have the appropriate roles to access it. Finally, all
read-only ports have been closed to not expose any cluster data. The only
remaining open ports without authentication are for healthz.
[1] https://bugs.launchpad.net/magnum/+bug/1758672
Loading…
Cancel
Save