[k8s] Add kubelet to the master nodes

Add kubelet on the master nodes. This work was
done already for calico, this patch applies the
same config when calico is used as well.

story: 2003521
task: 24797

Change-Id: Id33fb59ef23da740712d9a9b7ec4205bd6579b35

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@@ -6,14 +6,9 @@ echo "configuring kubernetes (master)"
 
 _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
 
-# TODO(flwang): We should revisit this part to figure out if it's possible to
+mkdir -p /opt/cni
-# only run the calico-node container as a systemd service before starting the
+_addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
-# minion nodes.
+atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
-if [ "$NETWORK_DRIVER" = "calico" ]; then
-    mkdir -p /opt/cni
-    _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
-    atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
-fi
 atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG}
 atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG}
 atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG}
@@ -131,11 +126,13 @@ if [ -n "${INSECURE_REGISTRY_URL}" ]; then
 fi
 
 if [ "$NETWORK_DRIVER" = "calico" ]; then
-    KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule"
+    KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
+fi
+KUBELET_ARGS="${KUBELET_ARGS} --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule"
 
-    KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml
+KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml
-    HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
+HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
-    cat << EOF >> ${KUBELET_KUBECONFIG}
+cat << EOF >> ${KUBELET_KUBECONFIG}
 apiVersion: v1
 clusters:
 - cluster:
@@ -158,7 +155,7 @@ users:
     client-key: ${CERT_DIR}/server.key
 EOF
 
-    cat > /etc/kubernetes/get_require_kubeconfig.sh <<EOF
+cat > /etc/kubernetes/get_require_kubeconfig.sh << EOF
 #!/bin/bash
 
 KUBE_VERSION=\$(kubelet --version | awk '{print \$2}')
@@ -167,37 +164,36 @@ if [[ "\${min_version}" != \$(echo -e "\${min_version}\n\${KUBE_VERSION}" | sort
     echo "--require-kubeconfig"
 fi
 EOF
-    chmod +x /etc/kubernetes/get_require_kubeconfig.sh
+chmod +x /etc/kubernetes/get_require_kubeconfig.sh
 
-    KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key --kubeconfig ${KUBELET_KUBECONFIG}"
+KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key --kubeconfig ${KUBELET_KUBECONFIG}"
 
-    # specified cgroup driver
+# specified cgroup driver
-    KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}"
+KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}"
 
-    systemctl disable docker
+systemctl disable docker
-    if cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then
+if cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then
-            cp /usr/lib/systemd/system/docker.service /etc/systemd/system/
+        cp /usr/lib/systemd/system/docker.service /etc/systemd/system/
-            sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \
+        sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \
-                    /etc/systemd/system/docker.service
+                /etc/systemd/system/docker.service
-    else
+else
-            cat > /etc/systemd/system/docker.service.d/cgroupdriver.conf << EOF
+        cat > /etc/systemd/system/docker.service.d/cgroupdriver.conf << EOF
 ExecStart=---exec-opt native.cgroupdriver=$CGROUP_DRIVER
 EOF
 
-    fi
-
-    systemctl daemon-reload
-    systemctl enable docker
-
-    if [ -z "${KUBE_NODE_IP}" ]; then
-        KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
-    fi
-
-    KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
-
-    sed -i '
-    /^KUBELET_ADDRESS=/ s/=.*/="--address=${KUBE_NODE_IP}"/
-    /^KUBELET_HOSTNAME=/ s/=.*/=""/
-    /^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
-' /etc/kubernetes/kubelet
 fi
+
+systemctl daemon-reload
+systemctl enable docker
+
+if [ -z "${KUBE_NODE_IP}" ]; then
+    KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
+fi
+
+KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
+
+sed -i '
+/^KUBELET_ADDRESS=/ s/=.*/="--address=${KUBE_NODE_IP}"/
+/^KUBELET_HOSTNAME=/ s/=.*/=""/
+/^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
+' /etc/kubernetes/kubelet

magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh

@@ -14,14 +14,8 @@ while [ ! -f /etc/kubernetes/certs/ca.key ] && \
 done
 
 echo "starting services"
-for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler kube-proxy; do
+for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler kubelet kube-proxy; do
     echo "activating service $service"
     systemctl enable $service
     systemctl --no-block start $service
 done
-
-if [ "$NETWORK_DRIVER" = "calico" ]; then
-    echo "activating service kubelet"
-    systemctl enable kubelet
-    systemctl start kubelet
-fi

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -558,6 +558,12 @@ resources:
       group: ungrouped
       config: {get_file: ../../common/templates/kubernetes/fragments/flannel-config-service.sh}
 
+  flannel_service:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: ungrouped
+      config: {get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh}
+
   enable_services:
     type: OS::Heat::SoftwareConfig
     properties:
@@ -611,6 +617,7 @@ resources:
         - config: {get_resource: enable_services}
         - config: {get_resource: write_flannel_config}
         - config: {get_resource: flannel_config_service}
+        - config: {get_resource: flannel_service}
         - config: {get_resource: kube_apiserver_to_kubelet_role}
         - config: {get_resource: master_wc_notify}
 

releasenotes/notes/add-kubelet-to-master-nodes-da2d4ea0d3a332cd.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Deploy kubelet in master nodes for the k8s_fedora_atomic driver.
+    Previously it was done only for calico, now kubelet will run in all
+    cases. Really useful, for monitoing the master nodes (eg deploy fluentd)
+    or run the kubernetes control-plance self-hosted.