From 79c002ce7ac97c051c52d4e5aaf83353edd293fb Mon Sep 17 00:00:00 2001 From: Feilong Wang Date: Tue, 27 Feb 2018 16:56:31 +1300 Subject: [PATCH] Add calico-node on k8s master node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By current design, pods under kube-system will run on minion nodes. And given now we're not running kubelet on master node, so calico-node is not running on k8s master node. As a result, kubectl proxy is not working to access dashboard. And it's confirmed with calico team that the calico-node container must be running on master node if user want to use kubectl proxy, see [1]. So, the solution is enabling kubelet on master but disallow the other pods scheduled on master with taint/tolerations. Besides, this patch includes another fix about running calico on Fedora Atomic. Because Fedora Atomic is using NetworkManager, it manipulates the routing table for interfaces in the default network namespace where Calico veth pairs are anchored for connections to containers. This can interfere with the Calico agent’s ability to route correctly. Please see more information about this at [2]. [1] https://docs.projectcalico.org/v3.0/getting-started/kubernetes/ installation/integration#about-the-calico-components [2] https://docs.projectcalico.org/master/usage/troubleshooting/ #configure-networkmanager Closes-Bug: #1751978 Change-Id: Iacd964806a28b3ca6ba3e037c60060f0957d44aa --- .../kubernetes/fragments/calico-service.sh | 12 ++- .../fragments/configure-kubernetes-master.sh | 68 +++++++++++++- .../fragments/configure-kubernetes-minion.sh | 13 +++ .../fragments/enable-services-master.sh | 6 ++ .../kubernetes/fragments/make-cert.sh | 90 ++++++++++++------- .../templates/kubecluster.yaml | 3 + 6 files changed, 156 insertions(+), 36 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh index 8ff71f934c..1824bf74f1 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh @@ -107,11 +107,17 @@ spec: k8s-app: calico-node annotations: scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: | - [{"key": "dedicated", "value": "master", "effect": "NoSchedule" }, - {"key":"CriticalAddonsOnly", "operator":"Exists"}] spec: hostNetwork: true + tolerations: + # Make sure calico/node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists serviceAccountName: calico-node # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index d07e62df89..b195157b7c 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -5,6 +5,15 @@ echo "configuring kubernetes (master)" _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/} + +# TODO(flwang): We should revisit this part to figure out if it's possible to +# only run the calico-node container as a systemd service before starting the +# minion nodes. +if [ "$NETWORK_DRIVER" = "calico" ]; then + mkdir -p /opt/cni + _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}' + atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG} +fi atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG} @@ -74,7 +83,7 @@ sed -i ' sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//') -KUBELET_ARGS="--register-node=true --register-schedulable=false --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${HOSTNAME_OVERRIDE}" +KUBELET_ARGS="--register-node=true --register-schedulable=false --pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --hostname-override=${HOSTNAME_OVERRIDE}" KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}" KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}" @@ -84,3 +93,60 @@ sed -i 's/\-\-log\-driver\=journald//g' /etc/sysconfig/docker if [ -n "${INSECURE_REGISTRY_URL}" ]; then echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker fi + +if [ "$NETWORK_DRIVER" = "calico" ]; then + KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule" + + KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml + HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//') + cat << EOF >> ${KUBELET_KUBECONFIG} +apiVersion: v1 +clusters: +- cluster: + certificate-authority: ${CERT_DIR}/ca.crt + server: http://127.0.0.1:8080 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: system:node:${HOSTNAME_OVERRIDE} + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: system:node:${HOSTNAME_OVERRIDE} + user: + as-user-extra: {} + client-certificate: ${CERT_DIR}/server.crt + client-key: ${CERT_DIR}/server.key +EOF + + cat > /etc/kubernetes/get_require_kubeconfig.sh < ${CALICO_NM} +[keyfile] +unmanaged-devices=interface-name:cali*;interface-name:tunl* +EOF +} + systemctl restart NetworkManager + fi fi atomic install --storage ostree --system --system-package=no --set=ADDTL_MOUNTS=${_addtl_mounts} --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG} diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh index 1259ed3886..0db1cef0da 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh @@ -19,3 +19,9 @@ for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler systemctl enable $service systemctl --no-block start $service done + +if [ "$NETWORK_DRIVER" = "calico" ]; then + echo "activating service kubelet" + systemctl enable kubelet + systemctl start kubelet +fi \ No newline at end of file diff --git a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh index b9d8250cc2..748c10d231 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh @@ -65,14 +65,16 @@ sans="${sans},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,D cert_dir=/etc/kubernetes/certs mkdir -p "$cert_dir" - CA_CERT=$cert_dir/ca.crt -SERVER_CERT=$cert_dir/server.crt -SERVER_CSR=$cert_dir/server.csr -SERVER_KEY=$cert_dir/server.key -#Get a token by user credentials and trust -auth_json=$(cat << EOF +function generate_certificates { + _CERT=$cert_dir/${1}.crt + _CSR=$cert_dir/${1}.csr + _KEY=$cert_dir/${1}.key + _CONF=$2 + + #Get a token by user credentials and trust + auth_json=$(cat << EOF { "auth": { "identity": { @@ -91,16 +93,35 @@ auth_json=$(cat << EOF EOF ) -content_type='Content-Type: application/json' -url="$AUTH_URL/auth/tokens" -USER_TOKEN=`curl $VERIFY_CA -s -i -X POST -H "$content_type" -d "$auth_json" $url \ - | grep X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'` + content_type='Content-Type: application/json' + url="$AUTH_URL/auth/tokens" + USER_TOKEN=`curl $VERIFY_CA -s -i -X POST -H "$content_type" -d "$auth_json" $url \ + | grep X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'` -# Get CA certificate for this cluster -curl $VERIFY_CA -X GET \ - -H "X-Auth-Token: $USER_TOKEN" \ - -H "OpenStack-API-Version: container-infra latest" \ - $MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT} + # Get CA certificate for this cluster + curl $VERIFY_CA -X GET \ + -H "X-Auth-Token: $USER_TOKEN" \ + -H "OpenStack-API-Version: container-infra latest" \ + $MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT} + + # Generate server's private key and csr + openssl genrsa -out "${_KEY}" 4096 + chmod 400 "${_KEY}" + openssl req -new -days 1000 \ + -key "${_KEY}" \ + -out "${_CSR}" \ + -reqexts req_ext \ + -config "${_CONF}" + + # Send csr to Magnum to have it signed + csr_req=$(python -c "import json; fp = open('${_CSR}'); print json.dumps({'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()}); fp.close()") + curl $VERIFY_CA -X POST \ + -H "X-Auth-Token: $USER_TOKEN" \ + -H "OpenStack-API-Version: container-infra latest" \ + -H "Content-Type: application/json" \ + -d "$csr_req" \ + $MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${_CERT} +} # Create config for server's csr cat > ${cert_dir}/server.conf < ${cert_dir}/kubelet.conf < ${SERVER_CERT} +generate_certificates server ${cert_dir}/server.conf +generate_certificates kubelet ${cert_dir}/kubelet.conf # Common certs and key are created for both etcd and kubernetes services. # Both etcd and kube user should have permission to access the certs and key. @@ -140,6 +166,6 @@ usermod -a -G kube_etcd etcd usermod -a -G kube_etcd kube chmod 550 "${cert_dir}" chown -R kube:kube_etcd "${cert_dir}" -chmod 440 $SERVER_KEY +chmod 440 $cert_dir/server.key mkdir -p /etc/etcd/certs cp ${cert_dir}/* /etc/etcd/certs diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index d3fcd697f2..8d339532e5 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -527,6 +527,9 @@ resources: - protocol: tcp port_range_min: 6443 port_range_max: 6443 + - protocol: tcp + port_range_min: 10250 + port_range_max: 10250 - protocol: tcp port_range_min: 30000 port_range_max: 32767