[k8s] Add kubelet to the master nodes

Add kubelet on the master nodes. This work was
done already for calico, this patch applies the
same config when calico is used as well.

story: 2003521
task: 24797

Change-Id: Id33fb59ef23da740712d9a9b7ec4205bd6579b35
This commit is contained in:
Spyros Trigazis 2018-08-27 20:53:12 +02:00
parent ee643b3ccb
commit 6390e0dbd3
4 changed files with 52 additions and 48 deletions

View File

@ -6,14 +6,9 @@ echo "configuring kubernetes (master)"
_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/} _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
# TODO(flwang): We should revisit this part to figure out if it's possible to mkdir -p /opt/cni
# only run the calico-node container as a systemd service before starting the _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
# minion nodes. atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
if [ "$NETWORK_DRIVER" = "calico" ]; then
mkdir -p /opt/cni
_addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
fi
atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG}
atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG}
atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG}
@ -131,11 +126,13 @@ if [ -n "${INSECURE_REGISTRY_URL}" ]; then
fi fi
if [ "$NETWORK_DRIVER" = "calico" ]; then if [ "$NETWORK_DRIVER" = "calico" ]; then
KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule" KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
fi
KUBELET_ARGS="${KUBELET_ARGS} --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule"
KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml
HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//') HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
cat << EOF >> ${KUBELET_KUBECONFIG} cat << EOF >> ${KUBELET_KUBECONFIG}
apiVersion: v1 apiVersion: v1
clusters: clusters:
- cluster: - cluster:
@ -158,7 +155,7 @@ users:
client-key: ${CERT_DIR}/server.key client-key: ${CERT_DIR}/server.key
EOF EOF
cat > /etc/kubernetes/get_require_kubeconfig.sh <<EOF cat > /etc/kubernetes/get_require_kubeconfig.sh << EOF
#!/bin/bash #!/bin/bash
KUBE_VERSION=\$(kubelet --version | awk '{print \$2}') KUBE_VERSION=\$(kubelet --version | awk '{print \$2}')
@ -167,37 +164,36 @@ if [[ "\${min_version}" != \$(echo -e "\${min_version}\n\${KUBE_VERSION}" | sort
echo "--require-kubeconfig" echo "--require-kubeconfig"
fi fi
EOF EOF
chmod +x /etc/kubernetes/get_require_kubeconfig.sh chmod +x /etc/kubernetes/get_require_kubeconfig.sh
KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key --kubeconfig ${KUBELET_KUBECONFIG}" KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key --kubeconfig ${KUBELET_KUBECONFIG}"
# specified cgroup driver # specified cgroup driver
KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}"
systemctl disable docker systemctl disable docker
if cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then if cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then
cp /usr/lib/systemd/system/docker.service /etc/systemd/system/ cp /usr/lib/systemd/system/docker.service /etc/systemd/system/
sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \ sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \
/etc/systemd/system/docker.service /etc/systemd/system/docker.service
else else
cat > /etc/systemd/system/docker.service.d/cgroupdriver.conf << EOF cat > /etc/systemd/system/docker.service.d/cgroupdriver.conf << EOF
ExecStart=---exec-opt native.cgroupdriver=$CGROUP_DRIVER ExecStart=---exec-opt native.cgroupdriver=$CGROUP_DRIVER
EOF EOF
fi
systemctl daemon-reload
systemctl enable docker
if [ -z "${KUBE_NODE_IP}" ]; then
KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
fi
KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
sed -i '
/^KUBELET_ADDRESS=/ s/=.*/="--address=${KUBE_NODE_IP}"/
/^KUBELET_HOSTNAME=/ s/=.*/=""/
/^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
' /etc/kubernetes/kubelet
fi fi
systemctl daemon-reload
systemctl enable docker
if [ -z "${KUBE_NODE_IP}" ]; then
KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
fi
KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
sed -i '
/^KUBELET_ADDRESS=/ s/=.*/="--address=${KUBE_NODE_IP}"/
/^KUBELET_HOSTNAME=/ s/=.*/=""/
/^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
' /etc/kubernetes/kubelet

View File

@ -14,14 +14,8 @@ while [ ! -f /etc/kubernetes/certs/ca.key ] && \
done done
echo "starting services" echo "starting services"
for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler kube-proxy; do for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler kubelet kube-proxy; do
echo "activating service $service" echo "activating service $service"
systemctl enable $service systemctl enable $service
systemctl --no-block start $service systemctl --no-block start $service
done done
if [ "$NETWORK_DRIVER" = "calico" ]; then
echo "activating service kubelet"
systemctl enable kubelet
systemctl start kubelet
fi

View File

@ -558,6 +558,12 @@ resources:
group: ungrouped group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/flannel-config-service.sh} config: {get_file: ../../common/templates/kubernetes/fragments/flannel-config-service.sh}
flannel_service:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh}
enable_services: enable_services:
type: OS::Heat::SoftwareConfig type: OS::Heat::SoftwareConfig
properties: properties:
@ -611,6 +617,7 @@ resources:
- config: {get_resource: enable_services} - config: {get_resource: enable_services}
- config: {get_resource: write_flannel_config} - config: {get_resource: write_flannel_config}
- config: {get_resource: flannel_config_service} - config: {get_resource: flannel_config_service}
- config: {get_resource: flannel_service}
- config: {get_resource: kube_apiserver_to_kubelet_role} - config: {get_resource: kube_apiserver_to_kubelet_role}
- config: {get_resource: master_wc_notify} - config: {get_resource: master_wc_notify}

View File

@ -0,0 +1,7 @@
---
features:
- |
Deploy kubelet in master nodes for the k8s_fedora_atomic driver.
Previously it was done only for calico, now kubelet will run in all
cases. Really useful, for monitoing the master nodes (eg deploy fluentd)
or run the kubernetes control-plance self-hosted.