Container Infrastructure Management Service for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

269 lines
10 KiB

  1. #!/bin/sh -x
  2. . /etc/sysconfig/heat-params
  3. echo "configuring kubernetes (master)"
  4. if [ ! -z "$HTTP_PROXY" ]; then
  5. export HTTP_PROXY
  6. fi
  7. if [ ! -z "$HTTPS_PROXY" ]; then
  8. export HTTPS_PROXY
  9. fi
  10. if [ ! -z "$NO_PROXY" ]; then
  11. export NO_PROXY
  12. fi
  13. _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
  14. rm -rf /etc/cni/net.d/*
  15. rm -rf /var/lib/cni/*
  16. rm -rf /opt/cni/*
  17. mkdir -p /opt/cni
  18. mkdir -p /etc/cni/net.d/
  19. _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]},{"type":"bind","source":"/var/lib/docker","destination":"/var/lib/docker","options":["bind","rw","slave","mode=755"]}'
  20. if [ "$NETWORK_DRIVER" = "calico" ]; then
  21. if [ "`systemctl status NetworkManager.service | grep -o "Active: active"`" = "Active: active" ]; then
  22. CALICO_NM=/etc/NetworkManager/conf.d/calico.conf
  23. [ -f ${CALICO_NM} ] || {
  24. echo "Writing File: $CALICO_NM"
  25. mkdir -p $(dirname ${CALICO_NM})
  26. cat << EOF > ${CALICO_NM}
  27. [keyfile]
  28. unmanaged-devices=interface-name:cali*;interface-name:tunl*
  29. EOF
  30. }
  31. systemctl restart NetworkManager
  32. fi
  33. fi
  34. atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
  35. atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG}
  36. atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG}
  37. atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG}
  38. atomic install --storage ostree --system --system-package=no --name=kube-proxy ${_prefix}kubernetes-proxy:${KUBE_TAG}
  39. CERT_DIR=/etc/kubernetes/certs
  40. # kube-proxy config
  41. PROXY_KUBECONFIG=/etc/kubernetes/proxy-kubeconfig.yaml
  42. cat > /etc/kubernetes/proxy << EOF
  43. KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR}"
  44. EOF
  45. cat > ${PROXY_KUBECONFIG} << EOF
  46. apiVersion: v1
  47. clusters:
  48. - cluster:
  49. certificate-authority: ${CERT_DIR}/ca.crt
  50. server: http://127.0.0.1:8080
  51. name: kubernetes
  52. contexts:
  53. - context:
  54. cluster: kubernetes
  55. user: kube-proxy
  56. name: default
  57. current-context: default
  58. kind: Config
  59. preferences: {}
  60. users:
  61. - name: kube-proxy
  62. user:
  63. as-user-extra: {}
  64. EOF
  65. if [ "$NETWORK_DRIVER" = "flannel" ]; then
  66. atomic install --storage ostree --system --system-package=no \
  67. --name=flanneld ${_prefix}flannel:${FLANNEL_TAG}
  68. fi
  69. sed -i '
  70. /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/
  71. /^KUBE_MASTER=/ s|=.*|="--master=http://127.0.0.1:8080"|
  72. ' /etc/kubernetes/config
  73. KUBE_API_ARGS="--runtime-config=api/all=true"
  74. KUBE_API_ARGS="$KUBE_API_ARGS --allow-privileged=$KUBE_ALLOW_PRIV"
  75. KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP"
  76. KUBE_API_ARGS="$KUBE_API_ARGS $KUBEAPI_OPTIONS"
  77. if [ "$TLS_DISABLED" == "True" ]; then
  78. KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0 --insecure-port=$KUBE_API_PORT"
  79. else
  80. KUBE_API_ADDRESS="--bind-address=0.0.0.0 --secure-port=$KUBE_API_PORT"
  81. # insecure port is used internaly
  82. KUBE_API_ADDRESS="$KUBE_API_ADDRESS --insecure-bind-address=127.0.0.1 --insecure-port=8080"
  83. KUBE_API_ARGS="$KUBE_API_ARGS --authorization-mode=Node,RBAC --tls-cert-file=$CERT_DIR/server.crt"
  84. KUBE_API_ARGS="$KUBE_API_ARGS --tls-private-key-file=$CERT_DIR/server.key"
  85. KUBE_API_ARGS="$KUBE_API_ARGS --client-ca-file=$CERT_DIR/ca.crt"
  86. KUBE_API_ARGS="$KUBE_API_ARGS --service-account-key-file=${CERT_DIR}/service_account.key"
  87. KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-certificate-authority=${CERT_DIR}/ca.crt --kubelet-client-certificate=${CERT_DIR}/server.crt --kubelet-client-key=${CERT_DIR}/server.key --kubelet-https=true"
  88. fi
  89. KUBE_ADMISSION_CONTROL=""
  90. if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
  91. KUBE_ADMISSION_CONTROL="--admission-control=NodeRestriction,${ADMISSION_CONTROL_LIST}"
  92. fi
  93. if [ -n "$TRUST_ID" ] && [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
  94. KUBE_API_ARGS="$KUBE_API_ARGS --cloud-provider=external"
  95. fi
  96. if [ "$KEYSTONE_AUTH_ENABLED" == "True" ]; then
  97. KEYSTONE_WEBHOOK_CONFIG=/etc/kubernetes/keystone_webhook_config.yaml
  98. [ -f ${KEYSTONE_WEBHOOK_CONFIG} ] || {
  99. echo "Writing File: $KEYSTONE_WEBHOOK_CONFIG"
  100. mkdir -p $(dirname ${KEYSTONE_WEBHOOK_CONFIG})
  101. cat << EOF > ${KEYSTONE_WEBHOOK_CONFIG}
  102. ---
  103. apiVersion: v1
  104. kind: Config
  105. preferences: {}
  106. clusters:
  107. - cluster:
  108. insecure-skip-tls-verify: true
  109. server: https://127.0.0.1:8443/webhook
  110. name: webhook
  111. users:
  112. - name: webhook
  113. contexts:
  114. - context:
  115. cluster: webhook
  116. user: webhook
  117. name: webhook
  118. current-context: webhook
  119. EOF
  120. }
  121. KUBE_API_ARGS="$KUBE_API_ARGS --authentication-token-webhook-config-file=/etc/kubernetes/keystone_webhook_config.yaml --authorization-webhook-config-file=/etc/kubernetes/keystone_webhook_config.yaml"
  122. webhook_auth="--authorization-mode=Node,Webhook,RBAC"
  123. KUBE_API_ARGS=${KUBE_API_ARGS/--authorization-mode=Node,RBAC/$webhook_auth}
  124. fi
  125. sed -i '
  126. /^KUBE_API_ADDRESS=/ s/=.*/="'"${KUBE_API_ADDRESS}"'"/
  127. /^KUBE_SERVICE_ADDRESSES=/ s|=.*|="--service-cluster-ip-range='"$PORTAL_NETWORK_CIDR"'"|
  128. /^KUBE_API_ARGS=/ s|=.*|="'"${KUBE_API_ARGS}"'"|
  129. /^KUBE_ETCD_SERVERS=/ s/=.*/="--etcd-servers=http:\/\/127.0.0.1:2379"/
  130. /^KUBE_ADMISSION_CONTROL=/ s/=.*/="'"${KUBE_ADMISSION_CONTROL}"'"/
  131. ' /etc/kubernetes/apiserver
  132. # Add controller manager args
  133. KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true"
  134. KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-name=${CLUSTER_UUID}"
  135. KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS $KUBECONTROLLER_OPTIONS"
  136. if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
  137. KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/service_account_private.key --root-ca-file=$CERT_DIR/ca.crt"
  138. fi
  139. if [ -n "$TRUST_ID" ] && [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
  140. KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cloud-provider=external"
  141. KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --external-cloud-volume-plugin=openstack --cloud-config=/etc/kubernetes/cloud-config"
  142. fi
  143. if [ "$(echo $CERT_MANAGER_API | tr '[:upper:]' '[:lower:]')" = "true" ]; then
  144. KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-signing-cert-file=$CERT_DIR/ca.crt --cluster-signing-key-file=$CERT_DIR/ca.key"
  145. fi
  146. sed -i '
  147. /^KUBELET_ADDRESSES=/ s/=.*/="--machines='""'"/
  148. /^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"#
  149. ' /etc/kubernetes/controller-manager
  150. sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler
  151. mkdir -p /etc/kubernetes/manifests
  152. KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --hostname-override=${INSTANCE_NAME}"
  153. KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}pause:3.0"
  154. KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
  155. KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
  156. KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}"
  157. if [ -n "$TRUST_ID" ] && [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
  158. KUBELET_ARGS="${KUBELET_ARGS} --cloud-provider=external"
  159. fi
  160. # For using default log-driver, other options should be ignored
  161. sed -i 's/\-\-log\-driver\=journald//g' /etc/sysconfig/docker
  162. if [ -n "${INSECURE_REGISTRY_URL}" ]; then
  163. echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker
  164. fi
  165. if [ "$NETWORK_DRIVER" = "calico" ]; then
  166. KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
  167. fi
  168. KUBELET_ARGS="${KUBELET_ARGS} --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule"
  169. KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml
  170. cat << EOF >> ${KUBELET_KUBECONFIG}
  171. apiVersion: v1
  172. clusters:
  173. - cluster:
  174. certificate-authority: ${CERT_DIR}/ca.crt
  175. server: http://127.0.0.1:8080
  176. name: kubernetes
  177. contexts:
  178. - context:
  179. cluster: kubernetes
  180. user: system:node:${INSTANCE_NAME}
  181. name: default
  182. current-context: default
  183. kind: Config
  184. preferences: {}
  185. users:
  186. - name: system:node:${INSTANCE_NAME}
  187. user:
  188. as-user-extra: {}
  189. client-certificate: ${CERT_DIR}/server.crt
  190. client-key: ${CERT_DIR}/server.key
  191. EOF
  192. cat > /etc/kubernetes/get_require_kubeconfig.sh << EOF
  193. #!/bin/bash
  194. KUBE_VERSION=\$(kubelet --version | awk '{print \$2}')
  195. min_version=v1.8.0
  196. if [[ "\${min_version}" != \$(echo -e "\${min_version}\n\${KUBE_VERSION}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) && "\${KUBE_VERSION}" != "devel" ]]; then
  197. echo "--require-kubeconfig"
  198. fi
  199. EOF
  200. chmod +x /etc/kubernetes/get_require_kubeconfig.sh
  201. KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key --kubeconfig ${KUBELET_KUBECONFIG}"
  202. # specified cgroup driver
  203. KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}"
  204. systemctl disable docker
  205. if cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then
  206. cp /usr/lib/systemd/system/docker.service /etc/systemd/system/
  207. sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \
  208. /etc/systemd/system/docker.service
  209. else
  210. cat > /etc/systemd/system/docker.service.d/cgroupdriver.conf << EOF
  211. ExecStart=---exec-opt native.cgroupdriver=$CGROUP_DRIVER
  212. EOF
  213. fi
  214. systemctl daemon-reload
  215. systemctl enable docker
  216. if [ -z "${KUBE_NODE_IP}" ]; then
  217. KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
  218. fi
  219. KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
  220. sed -i '
  221. /^KUBELET_ADDRESS=/ s/=.*/="--address=${KUBE_NODE_IP}"/
  222. /^KUBELET_HOSTNAME=/ s/=.*/=""/
  223. /^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
  224. ' /etc/kubernetes/kubelet