Container Infrastructure Management Service for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

286 lines
9.9 KiB

  1. #!/bin/bash
  2. set +x
  3. . /etc/sysconfig/heat-params
  4. set -x
  5. set -e
  6. ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
  7. echo "configuring kubernetes (minion)"
  8. if [ ! -z "$HTTP_PROXY" ]; then
  9. export HTTP_PROXY
  10. fi
  11. if [ ! -z "$HTTPS_PROXY" ]; then
  12. export HTTPS_PROXY
  13. fi
  14. if [ ! -z "$NO_PROXY" ]; then
  15. export NO_PROXY
  16. fi
  17. $ssh_cmd rm -rf /etc/cni/net.d/*
  18. $ssh_cmd rm -rf /var/lib/cni/*
  19. $ssh_cmd rm -rf /opt/cni/*
  20. $ssh_cmd mkdir -p /opt/cni
  21. $ssh_cmd mkdir -p /opt/cni/bin
  22. $ssh_cmd mkdir -p /etc/cni/net.d/
  23. _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]},{"type":"bind","source":"/var/lib/docker","destination":"/var/lib/docker","options":["bind","rw","slave","mode=755"]}'
  24. if [ "$NETWORK_DRIVER" = "calico" ]; then
  25. echo "net.ipv4.conf.all.rp_filter = 1" >> /etc/sysctl.conf
  26. $ssh_cmd sysctl -p
  27. if [ "$($ssh_cmd systemctl status NetworkManager.service | grep -o "Active: active")" = "Active: active" ]; then
  28. CALICO_NM=/etc/NetworkManager/conf.d/calico.conf
  29. [ -f ${CALICO_NM} ] || {
  30. echo "Writing File: $CALICO_NM"
  31. mkdir -p $(dirname ${CALICO_NM})
  32. cat << EOF > ${CALICO_NM}
  33. [keyfile]
  34. unmanaged-devices=interface-name:cali*;interface-name:tunl*
  35. EOF
  36. }
  37. $ssh_cmd systemctl restart NetworkManager
  38. fi
  39. elif [ "$NETWORK_DRIVER" = "flannel" ]; then
  40. $ssh_cmd modprobe vxlan
  41. echo "vxlan" > /etc/modules-load.d/vxlan.conf
  42. fi
  43. mkdir -p /srv/magnum/kubernetes/
  44. cat > /etc/kubernetes/config <<EOF
  45. KUBE_LOGTOSTDERR="--logtostderr=true"
  46. KUBE_LOG_LEVEL="--v=3"
  47. KUBE_MASTER="--master=http://127.0.0.1:8080"
  48. EOF
  49. cat > /etc/kubernetes/kubelet <<EOF
  50. KUBELET_ARGS="--fail-swap-on=false"
  51. EOF
  52. cat > /etc/kubernetes/proxy <<EOF
  53. KUBE_PROXY_ARGS=""
  54. EOF
  55. cat > /etc/systemd/system/kubelet.service <<EOF
  56. [Unit]
  57. Description=Kubelet via Hyperkube (System Container)
  58. [Service]
  59. EnvironmentFile=/etc/sysconfig/heat-params
  60. EnvironmentFile=/etc/kubernetes/config
  61. EnvironmentFile=/etc/kubernetes/kubelet
  62. ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
  63. ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
  64. ExecStartPre=/bin/mkdir -p /var/lib/calico
  65. ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
  66. ExecStartPre=/bin/mkdir -p /opt/cni/bin
  67. ExecStartPre=-/usr/bin/podman rm kubelet
  68. ExecStart=/bin/bash -c '/usr/bin/podman run --name kubelet \\
  69. --privileged \\
  70. --pid host \\
  71. --network host \\
  72. --volume /etc/cni/net.d:/etc/cni/net.d:ro,z \\
  73. --volume /etc/kubernetes:/etc/kubernetes:ro,z \\
  74. --volume /usr/lib/os-release:/etc/os-release:ro \\
  75. --volume /etc/ssl/certs:/etc/ssl/certs:ro \\
  76. --volume /lib/modules:/lib/modules:ro \\
  77. --volume /run:/run \\
  78. --volume /dev:/dev \\
  79. --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \\
  80. --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \\
  81. --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
  82. --volume /var/lib/calico:/var/lib/calico \\
  83. --volume /var/lib/docker:/var/lib/docker \\
  84. --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \\
  85. --volume /var/log:/var/log \\
  86. --volume /var/run:/var/run \\
  87. --volume /var/run/lock:/var/run/lock:z \\
  88. --volume /opt/cni/bin:/opt/cni/bin:z \\
  89. \${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
  90. /hyperkube kubelet \\
  91. \$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBELET_API_SERVER \$KUBELET_ADDRESS \$KUBELET_PORT \$KUBELET_HOSTNAME \$KUBELET_ARGS'
  92. ExecStop=-/usr/bin/podman stop kubelet
  93. Delegate=yes
  94. Restart=always
  95. RestartSec=10
  96. [Install]
  97. WantedBy=multi-user.target
  98. EOF
  99. cat > /etc/systemd/system/kube-proxy.service <<EOF
  100. [Unit]
  101. Description=kube-proxy via Hyperkube
  102. [Service]
  103. EnvironmentFile=/etc/sysconfig/heat-params
  104. EnvironmentFile=/etc/kubernetes/config
  105. EnvironmentFile=/etc/kubernetes/proxy
  106. ExecStartPre=/bin/mkdir -p /etc/kubernetes/
  107. ExecStartPre=-/usr/bin/podman rm kube-proxy
  108. ExecStart=/bin/bash -c '/usr/bin/podman run --name kube-proxy \\
  109. --privileged \\
  110. --net host \\
  111. --volume /etc/kubernetes:/etc/kubernetes:ro,z \\
  112. --volume /usr/lib/os-release:/etc/os-release:ro \\
  113. --volume /etc/ssl/certs:/etc/ssl/certs:ro \\
  114. --volume /run:/run \\
  115. --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \\
  116. --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \\
  117. --volume /lib/modules:/lib/modules:ro \\
  118. --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
  119. \${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
  120. /hyperkube kube-proxy \\
  121. \$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBE_MASTER \$KUBE_PROXY_ARGS'
  122. ExecStop=-/usr/bin/podman stop kube-proxy
  123. Delegate=yes
  124. Restart=always
  125. RestartSec=10
  126. [Install]
  127. WantedBy=multi-user.target
  128. EOF
  129. CERT_DIR=/etc/kubernetes/certs
  130. ETCD_SERVER_IP=${ETCD_SERVER_IP:-$KUBE_MASTER_IP}
  131. KUBE_PROTOCOL="https"
  132. KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml
  133. PROXY_KUBECONFIG=/etc/kubernetes/proxy-config.yaml
  134. if [ "$TLS_DISABLED" = "True" ]; then
  135. KUBE_PROTOCOL="http"
  136. fi
  137. KUBE_MASTER_URI="$KUBE_PROTOCOL://$KUBE_MASTER_IP:$KUBE_API_PORT"
  138. if [ -z "${KUBE_NODE_IP}" ]; then
  139. KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
  140. fi
  141. cat << EOF >> ${KUBELET_KUBECONFIG}
  142. apiVersion: v1
  143. clusters:
  144. - cluster:
  145. certificate-authority: ${CERT_DIR}/ca.crt
  146. server: ${KUBE_MASTER_URI}
  147. name: kubernetes
  148. contexts:
  149. - context:
  150. cluster: kubernetes
  151. user: system:node:${INSTANCE_NAME}
  152. name: default
  153. current-context: default
  154. kind: Config
  155. preferences: {}
  156. users:
  157. - name: system:node:${INSTANCE_NAME}
  158. user:
  159. as-user-extra: {}
  160. client-certificate: ${CERT_DIR}/kubelet.crt
  161. client-key: ${CERT_DIR}/kubelet.key
  162. EOF
  163. cat << EOF >> ${PROXY_KUBECONFIG}
  164. apiVersion: v1
  165. clusters:
  166. - cluster:
  167. certificate-authority: ${CERT_DIR}/ca.crt
  168. server: ${KUBE_MASTER_URI}
  169. name: kubernetes
  170. contexts:
  171. - context:
  172. cluster: kubernetes
  173. user: kube-proxy
  174. name: default
  175. current-context: default
  176. kind: Config
  177. preferences: {}
  178. users:
  179. - name: kube-proxy
  180. user:
  181. as-user-extra: {}
  182. client-certificate: ${CERT_DIR}/proxy.crt
  183. client-key: ${CERT_DIR}/proxy.key
  184. EOF
  185. if [ "$TLS_DISABLED" = "True" ]; then
  186. sed -i 's/^.*user:$//' ${KUBELET_KUBECONFIG}
  187. sed -i 's/^.*client-certificate.*$//' ${KUBELET_KUBECONFIG}
  188. sed -i 's/^.*client-key.*$//' ${KUBELET_KUBECONFIG}
  189. sed -i 's/^.*certificate-authority.*$//' ${KUBELET_KUBECONFIG}
  190. fi
  191. chmod 0640 ${KUBELET_KUBECONFIG}
  192. chmod 0640 ${PROXY_KUBECONFIG}
  193. sed -i '
  194. /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/
  195. /^KUBE_ETCD_SERVERS=/ s|=.*|="--etcd-servers=http://'"$ETCD_SERVER_IP"':2379"|
  196. /^KUBE_MASTER=/ s|=.*|="--master='"$KUBE_MASTER_URI"'"|
  197. ' /etc/kubernetes/config
  198. # NOTE: Kubernetes plugin for Openstack requires that the node name registered
  199. # in the kube-apiserver be the same as the Nova name of the instance, so that
  200. # the plugin can use the name to query for attributes such as IP, etc.
  201. # The hostname of the node is set to be the Nova name of the instance, and
  202. # the option --hostname-override for kubelet uses the hostname to register the node.
  203. # Using any other name will break the load balancer and cinder volume features.
  204. mkdir -p /etc/kubernetes/manifests
  205. KUBELET_ARGS="--pod-manifest-path=/etc/kubernetes/manifests --kubeconfig ${KUBELET_KUBECONFIG} --hostname-override=${INSTANCE_NAME}"
  206. KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
  207. KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
  208. KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
  209. KUBELET_ARGS="${KUBELET_ARGS} --node-labels=magnum.openstack.org/role=${NODEGROUP_ROLE}"
  210. KUBELET_ARGS="${KUBELET_ARGS} --node-labels=magnum.openstack.org/nodegroup=${NODEGROUP_NAME}"
  211. KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}"
  212. if [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
  213. KUBELET_ARGS="${KUBELET_ARGS} --cloud-provider=external"
  214. fi
  215. # For using default log-driver, other options should be ignored
  216. sed -i 's/\-\-log\-driver\=journald//g' /etc/sysconfig/docker
  217. KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}pause:3.0"
  218. if [ -n "${INSECURE_REGISTRY_URL}" ]; then
  219. echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker
  220. fi
  221. KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key"
  222. # specified cgroup driver
  223. KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}"
  224. auto_healing_enabled=$(echo ${AUTO_HEALING_ENABLED} | tr '[:upper:]' '[:lower:]')
  225. autohealing_controller=$(echo ${AUTO_HEALING_CONTROLLER} | tr '[:upper:]' '[:lower:]')
  226. if [[ "${auto_healing_enabled}" = "true" && "${autohealing_controller}" = "draino" ]]; then
  227. KUBELET_ARGS="${KUBELET_ARGS} --node-labels=draino-enabled=true"
  228. fi
  229. $ssh_cmd systemctl disable docker
  230. if $ssh_cmd cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then
  231. $ssh_cmd "cp /usr/lib/systemd/system/docker.service /etc/systemd/system/"
  232. sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \
  233. /etc/systemd/system/docker.service
  234. else
  235. cat > /etc/systemd/system/docker.service.d/cgroupdriver.conf << EOF
  236. ExecStart=---exec-opt native.cgroupdriver=$CGROUP_DRIVER
  237. EOF
  238. fi
  239. $ssh_cmd systemctl daemon-reload
  240. $ssh_cmd systemctl enable docker
  241. KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
  242. sed -i '
  243. /^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
  244. /^KUBELET_HOSTNAME=/ s/=.*/=""/
  245. s/^KUBELET_API_SERVER=.*$//
  246. /^KUBELET_ARGS=/ s|=.*|="'"${KUBELET_ARGS}"'"|
  247. ' /etc/kubernetes/kubelet
  248. KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR} --hostname-override=${INSTANCE_NAME}"
  249. cat > /etc/kubernetes/proxy << EOF
  250. KUBE_PROXY_ARGS="${KUBE_PROXY_ARGS} ${KUBEPROXY_OPTIONS}"
  251. EOF
  252. cat >> /etc/environment <<EOF
  253. KUBERNETES_MASTER=$KUBE_MASTER_URI
  254. EOF