Add calico-node on k8s master node

By current design, pods under kube-system will run on minion nodes. And
given now we're not running kubelet on master node, so calico-node is
not running on k8s master node. As a result, kubectl proxy is not
working to access dashboard. And it's confirmed with calico team that
the calico-node container must be running on master node if user want
to use kubectl proxy, see [1]. So, the solution is enabling kubelet
on master but disallow the other pods scheduled on master with
taint/tolerations.

Besides, this patch includes another fix about running calico on
Fedora Atomic. Because Fedora Atomic is using NetworkManager, it
manipulates the routing table for interfaces in the default network
namespace where Calico veth pairs are anchored for connections to
containers. This can interfere with the Calico agent’s ability to
route correctly. Please see more information about this at [2].

[1] https://docs.projectcalico.org/v3.0/getting-started/kubernetes/
    installation/integration#about-the-calico-components
[2] https://docs.projectcalico.org/master/usage/troubleshooting/
    #configure-networkmanager

Closes-Bug: #1751978

Change-Id: Iacd964806a28b3ca6ba3e037c60060f0957d44aa
changes/39/548139/13
Feilong Wang 5 years ago
parent d9e590bdc6
commit 79c002ce7a
  1. 12
      magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh
  2. 68
      magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
  3. 13
      magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
  4. 6
      magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh
  5. 94
      magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh
  6. 3
      magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@ -107,11 +107,17 @@ spec:
k8s-app: calico-node
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
scheduler.alpha.kubernetes.io/tolerations: |
[{"key": "dedicated", "value": "master", "effect": "NoSchedule" },
{"key":"CriticalAddonsOnly", "operator":"Exists"}]
spec:
hostNetwork: true
tolerations:
# Make sure calico/node gets scheduled on all nodes.
- effect: NoSchedule
operator: Exists
# Mark the pod as a critical add-on for rescheduling.
- key: CriticalAddonsOnly
operator: Exists
- effect: NoExecute
operator: Exists
serviceAccountName: calico-node
# Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
# deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.

@ -5,6 +5,15 @@
echo "configuring kubernetes (master)"
_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
# TODO(flwang): We should revisit this part to figure out if it's possible to
# only run the calico-node container as a systemd service before starting the
# minion nodes.
if [ "$NETWORK_DRIVER" = "calico" ]; then
mkdir -p /opt/cni
_addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
fi
atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG}
atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG}
atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG}
@ -74,7 +83,7 @@ sed -i '
sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler
HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
KUBELET_ARGS="--register-node=true --register-schedulable=false --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${HOSTNAME_OVERRIDE}"
KUBELET_ARGS="--register-node=true --register-schedulable=false --pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --hostname-override=${HOSTNAME_OVERRIDE}"
KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}"
@ -84,3 +93,60 @@ sed -i 's/\-\-log\-driver\=journald//g' /etc/sysconfig/docker
if [ -n "${INSECURE_REGISTRY_URL}" ]; then
echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker
fi
if [ "$NETWORK_DRIVER" = "calico" ]; then
KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule"
KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml
HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
cat << EOF >> ${KUBELET_KUBECONFIG}
apiVersion: v1
clusters:
- cluster:
certificate-authority: ${CERT_DIR}/ca.crt
server: http://127.0.0.1:8080
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: system:node:${HOSTNAME_OVERRIDE}
name: default
current-context: default
kind: Config
preferences: {}
users:
- name: system:node:${HOSTNAME_OVERRIDE}
user:
as-user-extra: {}
client-certificate: ${CERT_DIR}/server.crt
client-key: ${CERT_DIR}/server.key
EOF
cat > /etc/kubernetes/get_require_kubeconfig.sh <<EOF
#!/bin/bash
KUBE_VERSION=\$(kubelet --version | awk '{print \$2}')
min_version=v1.8.0
if [[ "\${min_version}" != \$(echo -e "\${min_version}\n\${KUBE_VERSION}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) && "\${KUBE_VERSION}" != "devel" ]]; then
echo "--require-kubeconfig"
fi
EOF
chmod +x /etc/kubernetes/get_require_kubeconfig.sh
KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key --kubeconfig ${KUBELET_KUBECONFIG}"
# specified cgroup driver
KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=systemd"
if [ -z "${KUBE_NODE_IP}" ]; then
KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
fi
KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
sed -i '
/^KUBELET_ADDRESS=/ s/=.*/="--address=${KUBE_NODE_IP}"/
/^KUBELET_HOSTNAME=/ s/=.*/=""/
/^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
' /etc/kubernetes/kubelet
fi

@ -10,6 +10,19 @@ _addtl_mounts=''
if [ "$NETWORK_DRIVER" = "calico" ]; then
mkdir -p /opt/cni
_addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
if [ "`systemctl status NetworkManager.service | grep -o "Active: active"`" = "Active: active" ]; then
CALICO_NM=/etc/NetworkManager/conf.d/calico.conf
[ -f ${CALICO_NM} ] || {
echo "Writing File: $CALICO_NM"
mkdir -p $(dirname ${CALICO_NM})
cat << EOF > ${CALICO_NM}
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:tunl*
EOF
}
systemctl restart NetworkManager
fi
fi
atomic install --storage ostree --system --system-package=no --set=ADDTL_MOUNTS=${_addtl_mounts} --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}

@ -19,3 +19,9 @@ for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler
systemctl enable $service
systemctl --no-block start $service
done
if [ "$NETWORK_DRIVER" = "calico" ]; then
echo "activating service kubelet"
systemctl enable kubelet
systemctl start kubelet
fi

@ -65,14 +65,16 @@ sans="${sans},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,D
cert_dir=/etc/kubernetes/certs
mkdir -p "$cert_dir"
CA_CERT=$cert_dir/ca.crt
SERVER_CERT=$cert_dir/server.crt
SERVER_CSR=$cert_dir/server.csr
SERVER_KEY=$cert_dir/server.key
#Get a token by user credentials and trust
auth_json=$(cat << EOF
function generate_certificates {
_CERT=$cert_dir/${1}.crt
_CSR=$cert_dir/${1}.csr
_KEY=$cert_dir/${1}.key
_CONF=$2
#Get a token by user credentials and trust
auth_json=$(cat << EOF
{
"auth": {
"identity": {
@ -91,16 +93,35 @@ auth_json=$(cat << EOF
EOF
)
content_type='Content-Type: application/json'
url="$AUTH_URL/auth/tokens"
USER_TOKEN=`curl $VERIFY_CA -s -i -X POST -H "$content_type" -d "$auth_json" $url \
| grep X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'`
# Get CA certificate for this cluster
curl $VERIFY_CA -X GET \
-H "X-Auth-Token: $USER_TOKEN" \
-H "OpenStack-API-Version: container-infra latest" \
$MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT}
content_type='Content-Type: application/json'
url="$AUTH_URL/auth/tokens"
USER_TOKEN=`curl $VERIFY_CA -s -i -X POST -H "$content_type" -d "$auth_json" $url \
| grep X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'`
# Get CA certificate for this cluster
curl $VERIFY_CA -X GET \
-H "X-Auth-Token: $USER_TOKEN" \
-H "OpenStack-API-Version: container-infra latest" \
$MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT}
# Generate server's private key and csr
openssl genrsa -out "${_KEY}" 4096
chmod 400 "${_KEY}"
openssl req -new -days 1000 \
-key "${_KEY}" \
-out "${_CSR}" \
-reqexts req_ext \
-config "${_CONF}"
# Send csr to Magnum to have it signed
csr_req=$(python -c "import json; fp = open('${_CSR}'); print json.dumps({'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()}); fp.close()")
curl $VERIFY_CA -X POST \
-H "X-Auth-Token: $USER_TOKEN" \
-H "OpenStack-API-Version: container-infra latest" \
-H "Content-Type: application/json" \
-d "$csr_req" \
$MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${_CERT}
}
# Create config for server's csr
cat > ${cert_dir}/server.conf <<EOF
@ -115,23 +136,28 @@ subjectAltName = ${sans}
extendedKeyUsage = clientAuth,serverAuth
EOF
# Generate server's private key and csr
openssl genrsa -out "${SERVER_KEY}" 4096
chmod 400 "${SERVER_KEY}"
openssl req -new -days 1000 \
-key "${SERVER_KEY}" \
-out "${SERVER_CSR}" \
-reqexts req_ext \
-config "${cert_dir}/server.conf"
# Send csr to Magnum to have it signed
csr_req=$(python -c "import json; fp = open('${SERVER_CSR}'); print json.dumps({'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()}); fp.close()")
curl $VERIFY_CA -X POST \
-H "X-Auth-Token: $USER_TOKEN" \
-H "OpenStack-API-Version: container-infra latest" \
-H "Content-Type: application/json" \
-d "$csr_req" \
$MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${SERVER_CERT}
#Kubelet Certs
INSTANCE_NAME=$(hostname --short | sed 's/\.novalocal//')
cat > ${cert_dir}/kubelet.conf <<EOF
[req]
distinguished_name = req_distinguished_name
req_extensions = req_ext
prompt = no
[req_distinguished_name]
CN = system:node:${INSTANCE_NAME}
O=system:nodes
OU=OpenStack/Magnum
C=US
ST=TX
L=Austin
[req_ext]
subjectAltName = ${sans}
keyUsage=critical,digitalSignature,keyEncipherment
extendedKeyUsage=clientAuth,serverAuth
EOF
generate_certificates server ${cert_dir}/server.conf
generate_certificates kubelet ${cert_dir}/kubelet.conf
# Common certs and key are created for both etcd and kubernetes services.
# Both etcd and kube user should have permission to access the certs and key.
@ -140,6 +166,6 @@ usermod -a -G kube_etcd etcd
usermod -a -G kube_etcd kube
chmod 550 "${cert_dir}"
chown -R kube:kube_etcd "${cert_dir}"
chmod 440 $SERVER_KEY
chmod 440 $cert_dir/server.key
mkdir -p /etc/etcd/certs
cp ${cert_dir}/* /etc/etcd/certs

@ -527,6 +527,9 @@ resources:
- protocol: tcp
port_range_min: 6443
port_range_max: 6443
- protocol: tcp
port_range_min: 10250
port_range_max: 10250
- protocol: tcp
port_range_min: 30000
port_range_max: 32767

Loading…
Cancel
Save