From 6390e0dbd3a20f71d1b17999e1f21774fbb1c27e Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Mon, 27 Aug 2018 20:53:12 +0200 Subject: [PATCH] [k8s] Add kubelet to the master nodes Add kubelet on the master nodes. This work was done already for calico, this patch applies the same config when calico is used as well. story: 2003521 task: 24797 Change-Id: Id33fb59ef23da740712d9a9b7ec4205bd6579b35 --- .../fragments/configure-kubernetes-master.sh | 76 +++++++++---------- .../fragments/enable-services-master.sh | 10 +-- .../templates/kubemaster.yaml | 7 ++ ...elet-to-master-nodes-da2d4ea0d3a332cd.yaml | 7 ++ 4 files changed, 52 insertions(+), 48 deletions(-) create mode 100644 releasenotes/notes/add-kubelet-to-master-nodes-da2d4ea0d3a332cd.yaml diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 6b21f93083..d9f30cd36a 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -6,14 +6,9 @@ echo "configuring kubernetes (master)" _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/} -# TODO(flwang): We should revisit this part to figure out if it's possible to -# only run the calico-node container as a systemd service before starting the -# minion nodes. -if [ "$NETWORK_DRIVER" = "calico" ]; then - mkdir -p /opt/cni - _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}' - atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG} -fi +mkdir -p /opt/cni +_addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}' +atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG} @@ -131,11 +126,13 @@ if [ -n "${INSECURE_REGISTRY_URL}" ]; then fi if [ "$NETWORK_DRIVER" = "calico" ]; then - KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule" + KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" +fi +KUBELET_ARGS="${KUBELET_ARGS} --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule" - KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml - HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//') - cat << EOF >> ${KUBELET_KUBECONFIG} +KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml +HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//') +cat << EOF >> ${KUBELET_KUBECONFIG} apiVersion: v1 clusters: - cluster: @@ -158,7 +155,7 @@ users: client-key: ${CERT_DIR}/server.key EOF - cat > /etc/kubernetes/get_require_kubeconfig.sh < /etc/kubernetes/get_require_kubeconfig.sh << EOF #!/bin/bash KUBE_VERSION=\$(kubelet --version | awk '{print \$2}') @@ -167,37 +164,36 @@ if [[ "\${min_version}" != \$(echo -e "\${min_version}\n\${KUBE_VERSION}" | sort echo "--require-kubeconfig" fi EOF - chmod +x /etc/kubernetes/get_require_kubeconfig.sh +chmod +x /etc/kubernetes/get_require_kubeconfig.sh - KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key --kubeconfig ${KUBELET_KUBECONFIG}" +KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key --kubeconfig ${KUBELET_KUBECONFIG}" - # specified cgroup driver - KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" +# specified cgroup driver +KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" - systemctl disable docker - if cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then - cp /usr/lib/systemd/system/docker.service /etc/systemd/system/ - sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \ - /etc/systemd/system/docker.service - else - cat > /etc/systemd/system/docker.service.d/cgroupdriver.conf << EOF +systemctl disable docker +if cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then + cp /usr/lib/systemd/system/docker.service /etc/systemd/system/ + sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \ + /etc/systemd/system/docker.service +else + cat > /etc/systemd/system/docker.service.d/cgroupdriver.conf << EOF ExecStart=---exec-opt native.cgroupdriver=$CGROUP_DRIVER EOF - fi - - systemctl daemon-reload - systemctl enable docker - - if [ -z "${KUBE_NODE_IP}" ]; then - KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) - fi - - KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true" - - sed -i ' - /^KUBELET_ADDRESS=/ s/=.*/="--address=${KUBE_NODE_IP}"/ - /^KUBELET_HOSTNAME=/ s/=.*/=""/ - /^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"| -' /etc/kubernetes/kubelet fi + +systemctl daemon-reload +systemctl enable docker + +if [ -z "${KUBE_NODE_IP}" ]; then + KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) +fi + +KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true" + +sed -i ' +/^KUBELET_ADDRESS=/ s/=.*/="--address=${KUBE_NODE_IP}"/ +/^KUBELET_HOSTNAME=/ s/=.*/=""/ +/^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"| +' /etc/kubernetes/kubelet diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh index 94e0d46841..310641de68 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh @@ -14,14 +14,8 @@ while [ ! -f /etc/kubernetes/certs/ca.key ] && \ done echo "starting services" -for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler kube-proxy; do +for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler kubelet kube-proxy; do echo "activating service $service" systemctl enable $service systemctl --no-block start $service -done - -if [ "$NETWORK_DRIVER" = "calico" ]; then - echo "activating service kubelet" - systemctl enable kubelet - systemctl start kubelet -fi \ No newline at end of file +done \ No newline at end of file diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 0f497066e0..7ed4b17ae5 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -558,6 +558,12 @@ resources: group: ungrouped config: {get_file: ../../common/templates/kubernetes/fragments/flannel-config-service.sh} + flannel_service: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: {get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh} + enable_services: type: OS::Heat::SoftwareConfig properties: @@ -611,6 +617,7 @@ resources: - config: {get_resource: enable_services} - config: {get_resource: write_flannel_config} - config: {get_resource: flannel_config_service} + - config: {get_resource: flannel_service} - config: {get_resource: kube_apiserver_to_kubelet_role} - config: {get_resource: master_wc_notify} diff --git a/releasenotes/notes/add-kubelet-to-master-nodes-da2d4ea0d3a332cd.yaml b/releasenotes/notes/add-kubelet-to-master-nodes-da2d4ea0d3a332cd.yaml new file mode 100644 index 0000000000..ac8ffd22e4 --- /dev/null +++ b/releasenotes/notes/add-kubelet-to-master-nodes-da2d4ea0d3a332cd.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Deploy kubelet in master nodes for the k8s_fedora_atomic driver. + Previously it was done only for calico, now kubelet will run in all + cases. Really useful, for monitoing the master nodes (eg deploy fluentd) + or run the kubernetes control-plance self-hosted.