diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh index 8ff71f934c..1824bf74f1 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh @@ -107,11 +107,17 @@ spec: k8s-app: calico-node annotations: scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: | - [{"key": "dedicated", "value": "master", "effect": "NoSchedule" }, - {"key":"CriticalAddonsOnly", "operator":"Exists"}] spec: hostNetwork: true + tolerations: + # Make sure calico/node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists serviceAccountName: calico-node # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index d07e62df89..b195157b7c 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -5,6 +5,15 @@ echo "configuring kubernetes (master)" _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/} + +# TODO(flwang): We should revisit this part to figure out if it's possible to +# only run the calico-node container as a systemd service before starting the +# minion nodes. +if [ "$NETWORK_DRIVER" = "calico" ]; then + mkdir -p /opt/cni + _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}' + atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG} +fi atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG} @@ -74,7 +83,7 @@ sed -i ' sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//') -KUBELET_ARGS="--register-node=true --register-schedulable=false --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${HOSTNAME_OVERRIDE}" +KUBELET_ARGS="--register-node=true --register-schedulable=false --pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --hostname-override=${HOSTNAME_OVERRIDE}" KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}" KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}" @@ -84,3 +93,60 @@ sed -i 's/\-\-log\-driver\=journald//g' /etc/sysconfig/docker if [ -n "${INSECURE_REGISTRY_URL}" ]; then echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker fi + +if [ "$NETWORK_DRIVER" = "calico" ]; then + KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule" + + KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml + HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//') + cat << EOF >> ${KUBELET_KUBECONFIG} +apiVersion: v1 +clusters: +- cluster: + certificate-authority: ${CERT_DIR}/ca.crt + server: http://127.0.0.1:8080 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: system:node:${HOSTNAME_OVERRIDE} + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: system:node:${HOSTNAME_OVERRIDE} + user: + as-user-extra: {} + client-certificate: ${CERT_DIR}/server.crt + client-key: ${CERT_DIR}/server.key +EOF + + cat > /etc/kubernetes/get_require_kubeconfig.sh < ${CALICO_NM} +[keyfile] +unmanaged-devices=interface-name:cali*;interface-name:tunl* +EOF +} + systemctl restart NetworkManager + fi fi atomic install --storage ostree --system --system-package=no --set=ADDTL_MOUNTS=${_addtl_mounts} --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG} diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh index 1259ed3886..0db1cef0da 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh @@ -19,3 +19,9 @@ for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler systemctl enable $service systemctl --no-block start $service done + +if [ "$NETWORK_DRIVER" = "calico" ]; then + echo "activating service kubelet" + systemctl enable kubelet + systemctl start kubelet +fi \ No newline at end of file diff --git a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh index b9d8250cc2..748c10d231 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh @@ -65,14 +65,16 @@ sans="${sans},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,D cert_dir=/etc/kubernetes/certs mkdir -p "$cert_dir" - CA_CERT=$cert_dir/ca.crt -SERVER_CERT=$cert_dir/server.crt -SERVER_CSR=$cert_dir/server.csr -SERVER_KEY=$cert_dir/server.key -#Get a token by user credentials and trust -auth_json=$(cat << EOF +function generate_certificates { + _CERT=$cert_dir/${1}.crt + _CSR=$cert_dir/${1}.csr + _KEY=$cert_dir/${1}.key + _CONF=$2 + + #Get a token by user credentials and trust + auth_json=$(cat << EOF { "auth": { "identity": { @@ -91,16 +93,35 @@ auth_json=$(cat << EOF EOF ) -content_type='Content-Type: application/json' -url="$AUTH_URL/auth/tokens" -USER_TOKEN=`curl $VERIFY_CA -s -i -X POST -H "$content_type" -d "$auth_json" $url \ - | grep X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'` + content_type='Content-Type: application/json' + url="$AUTH_URL/auth/tokens" + USER_TOKEN=`curl $VERIFY_CA -s -i -X POST -H "$content_type" -d "$auth_json" $url \ + | grep X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'` -# Get CA certificate for this cluster -curl $VERIFY_CA -X GET \ - -H "X-Auth-Token: $USER_TOKEN" \ - -H "OpenStack-API-Version: container-infra latest" \ - $MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT} + # Get CA certificate for this cluster + curl $VERIFY_CA -X GET \ + -H "X-Auth-Token: $USER_TOKEN" \ + -H "OpenStack-API-Version: container-infra latest" \ + $MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT} + + # Generate server's private key and csr + openssl genrsa -out "${_KEY}" 4096 + chmod 400 "${_KEY}" + openssl req -new -days 1000 \ + -key "${_KEY}" \ + -out "${_CSR}" \ + -reqexts req_ext \ + -config "${_CONF}" + + # Send csr to Magnum to have it signed + csr_req=$(python -c "import json; fp = open('${_CSR}'); print json.dumps({'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()}); fp.close()") + curl $VERIFY_CA -X POST \ + -H "X-Auth-Token: $USER_TOKEN" \ + -H "OpenStack-API-Version: container-infra latest" \ + -H "Content-Type: application/json" \ + -d "$csr_req" \ + $MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${_CERT} +} # Create config for server's csr cat > ${cert_dir}/server.conf < ${cert_dir}/kubelet.conf < ${SERVER_CERT} +generate_certificates server ${cert_dir}/server.conf +generate_certificates kubelet ${cert_dir}/kubelet.conf # Common certs and key are created for both etcd and kubernetes services. # Both etcd and kube user should have permission to access the certs and key. @@ -140,6 +166,6 @@ usermod -a -G kube_etcd etcd usermod -a -G kube_etcd kube chmod 550 "${cert_dir}" chown -R kube:kube_etcd "${cert_dir}" -chmod 440 $SERVER_KEY +chmod 440 $cert_dir/server.key mkdir -p /etc/etcd/certs cp ${cert_dir}/* /etc/etcd/certs diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index d3fcd697f2..8d339532e5 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -527,6 +527,9 @@ resources: - protocol: tcp port_range_min: 6443 port_range_max: 6443 + - protocol: tcp + port_range_min: 10250 + port_range_max: 10250 - protocol: tcp port_range_min: 30000 port_range_max: 32767