Repository for OpenStack Helm infrastructure-related code
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

005-deploy-k8s.sh 6.5KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201
  1. #!/bin/bash
  2. # Copyright 2017 The Openstack-Helm Authors.
  3. # Copyright 2019, AT&T Intellectual Property
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  6. # not use this file except in compliance with the License. You may obtain
  7. # a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14. # License for the specific language governing permissions and limitations
  15. # under the License.
  16. set -xe
  17. : ${HELM_VERSION:="v2.13.1"}
  18. : ${KUBE_VERSION:="v1.13.4"}
  19. : ${MINIKUBE_VERSION:="v0.30.0"}
  20. : ${CALICO_VERSION:="v3.3"}
  21. : "${HTTP_PROXY:=""}"
  22. : "${HTTPS_PROXY:=""}"
  23. export DEBCONF_NONINTERACTIVE_SEEN=true
  24. export DEBIAN_FRONTEND=noninteractive
  25. function configure_resolvconf {
  26. # Setup resolv.conf to use the k8s api server, which is required for the
  27. # kubelet to resolve cluster services.
  28. sudo mv /etc/resolv.conf /etc/resolv.conf.backup
  29. sudo bash -c "echo 'search svc.cluster.local cluster.local' > /etc/resolv.conf"
  30. sudo bash -c "echo 'nameserver 10.96.0.10' >> /etc/resolv.conf"
  31. # NOTE(drewwalters96): Use the Google DNS servers to prevent local addresses in
  32. # the resolv.conf file unless using a proxy, then use the existing DNS servers,
  33. # as custom DNS nameservers are commonly required when using a proxy server.
  34. if [ -z "${HTTP_PROXY}" ]; then
  35. sudo bash -c "echo 'nameserver 8.8.8.8' >> /etc/resolv.conf"
  36. sudo bash -c "echo 'nameserver 8.8.4.4' >> /etc/resolv.conf"
  37. else
  38. sed -ne "s/nameserver //p" /etc/resolv.conf.backup | while read -r ns; do
  39. sudo bash -c "echo 'nameserver ${ns}' >> /etc/resolv.conf"
  40. done
  41. fi
  42. sudo bash -c "echo 'options ndots:5 timeout:1 attempts:1' >> /etc/resolv.conf"
  43. sudo rm /etc/resolv.conf.backup
  44. }
  45. # NOTE: Clean Up hosts file
  46. sudo sed -i '/^127.0.0.1/c\127.0.0.1 localhost localhost.localdomain localhost4localhost4.localdomain4' /etc/hosts
  47. sudo sed -i '/^::1/c\::1 localhost6 localhost6.localdomain6' /etc/hosts
  48. # Install required packages for K8s on host
  49. wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
  50. RELEASE_NAME=$(grep 'CODENAME' /etc/lsb-release | awk -F= '{print $2}')
  51. sudo add-apt-repository "deb https://download.ceph.com/debian-mimic/
  52. ${RELEASE_NAME} main"
  53. sudo -E apt-get update
  54. # NOTE(srwilkers): Pin docker version to validated docker version for k8s 1.12.2
  55. sudo -E apt-get install -y \
  56. docker.io=18.06.1-0ubuntu1.2~18.04.1 \
  57. socat \
  58. jq \
  59. util-linux \
  60. ceph-common \
  61. rbd-nbd \
  62. nfs-common \
  63. bridge-utils \
  64. libxtables12
  65. sudo -E tee /etc/modprobe.d/rbd.conf << EOF
  66. install rbd /bin/true
  67. EOF
  68. configure_resolvconf
  69. # Install minikube and kubectl
  70. URL="https://storage.googleapis.com"
  71. sudo -E curl -sSLo /usr/local/bin/minikube \
  72. "${URL}"/minikube/releases/"${MINIKUBE_VERSION}"/minikube-linux-amd64
  73. sudo -E curl -sSLo /usr/local/bin/kubectl \
  74. "${URL}"/kubernetes-release/release/"${KUBE_VERSION}"/bin/linux/amd64/kubectl
  75. sudo -E chmod +x /usr/local/bin/minikube
  76. sudo -E chmod +x /usr/local/bin/kubectl
  77. # Install Helm
  78. TMP_DIR=$(mktemp -d)
  79. sudo -E bash -c \
  80. "curl -sSL ${URL}/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | \
  81. tar -zxv --strip-components=1 -C ${TMP_DIR}"
  82. sudo -E mv "${TMP_DIR}"/helm /usr/local/bin/helm
  83. rm -rf "${TMP_DIR}"
  84. # NOTE: Deploy kubenetes using minikube. A CNI that supports network policy is
  85. # required for validation; use calico for simplicity.
  86. sudo -E minikube config set embed-certs true
  87. sudo -E minikube config set kubernetes-version "${KUBE_VERSION}"
  88. sudo -E minikube config set vm-driver none
  89. sudo -E minikube addons disable addon-manager
  90. sudo -E minikube addons disable dashboard
  91. export CHANGE_MINIKUBE_NONE_USER=true
  92. sudo -E minikube start \
  93. --docker-env HTTP_PROXY="${HTTP_PROXY}" \
  94. --docker-env HTTPS_PROXY="${HTTPS_PROXY}" \
  95. --docker-env NO_PROXY="${NO_PROXY},10.96.0.0/12" \
  96. --extra-config=kubelet.network-plugin=cni \
  97. --extra-config=controller-manager.allocate-node-cidrs=true \
  98. --extra-config=controller-manager.cluster-cidr=192.168.0.0/16
  99. kubectl apply -f \
  100. https://docs.projectcalico.org/"${CALICO_VERSION}"/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
  101. kubectl apply -f \
  102. https://docs.projectcalico.org/"${CALICO_VERSION}"/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
  103. # NOTE: Wait for node to be ready.
  104. kubectl wait --timeout=240s --for=condition=Ready nodes/minikube
  105. # NOTE: Wait for dns to be running.
  106. END=$(($(date +%s) + 240))
  107. until kubectl --namespace=kube-system \
  108. get pods -l k8s-app=kube-dns --no-headers -o name | grep -q "^pod/coredns"; do
  109. NOW=$(date +%s)
  110. [ "${NOW}" -gt "${END}" ] && exit 1
  111. echo "still waiting for dns"
  112. sleep 10
  113. done
  114. kubectl --namespace=kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns
  115. # Deploy helm/tiller into the cluster
  116. kubectl create -n kube-system serviceaccount helm-tiller
  117. cat <<EOF | kubectl apply -f -
  118. apiVersion: rbac.authorization.k8s.io/v1
  119. kind: ClusterRoleBinding
  120. metadata:
  121. name: helm-tiller
  122. roleRef:
  123. apiGroup: rbac.authorization.k8s.io
  124. kind: ClusterRole
  125. name: cluster-admin
  126. subjects:
  127. - kind: ServiceAccount
  128. name: helm-tiller
  129. namespace: kube-system
  130. EOF
  131. helm init --service-account helm-tiller
  132. kubectl --namespace=kube-system wait \
  133. --timeout=240s \
  134. --for=condition=Ready \
  135. pod -l app=helm,name=tiller
  136. # Set up local helm server
  137. sudo -E tee /etc/systemd/system/helm-serve.service << EOF
  138. [Unit]
  139. Description=Helm Server
  140. After=network.target
  141. [Service]
  142. User=$(id -un 2>&1)
  143. Restart=always
  144. ExecStart=/usr/local/bin/helm serve
  145. [Install]
  146. WantedBy=multi-user.target
  147. EOF
  148. sudo chmod 0640 /etc/systemd/system/helm-serve.service
  149. sudo systemctl daemon-reload
  150. sudo systemctl restart helm-serve
  151. sudo systemctl enable helm-serve
  152. # Remove stable repo, if present, to improve build time
  153. helm repo remove stable || true
  154. # Set up local helm repo
  155. helm repo add local http://localhost:8879/charts
  156. helm repo update
  157. make
  158. # Set required labels on host(s)
  159. kubectl label nodes --all openstack-control-plane=enabled
  160. kubectl label nodes --all openstack-compute-node=enabled
  161. kubectl label nodes --all openvswitch=enabled
  162. kubectl label nodes --all linuxbridge=enabled
  163. kubectl label nodes --all ceph-mon=enabled
  164. kubectl label nodes --all ceph-osd=enabled
  165. kubectl label nodes --all ceph-mds=enabled
  166. kubectl label nodes --all ceph-rgw=enabled
  167. kubectl label nodes --all ceph-mgr=enabled