Support k8s installation

Add support for installing kubernetes cluster via devstack.
It uses kubeadm to bootstrap the k8s cluster.

Change-Id: I7877ceda08bbdab807116a13d74ff884136dc501
This commit is contained in:
Hongbin Lu 2019-03-23 20:55:16 +00:00
parent f89832fe0f
commit 80f8d7f260
4 changed files with 228 additions and 2 deletions

View File

@ -2,8 +2,8 @@
Container Plugin
================
This plugin enables installation of container engine on Devstack. The default
container engine is Docker (currently this plugin supports only Docker!).
This plugin enables installation of container engine and Kubernetes on
Devstack. The default container engine is Docker.
====================
Enabling in Devstack
@ -21,6 +21,9 @@ For more info on devstack installation follow the below link:
2. Add this repo as an external repository
------------------------------------------
This plugin supports installing Kubernetes or container engine only.
For installing container engine only, using the following config:
.. code-block:: ini
cat > /opt/stack/devstack/local.conf << END
@ -28,6 +31,41 @@ For more info on devstack installation follow the below link:
enable_plugin devstack-plugin-container https://opendev.org/openstack/devstack-plugin-container
END
For installing Kubernetes, using the following config in master node:
.. code-block:: ini
cat > /opt/stack/devstack/local.conf << END
[[local|localrc]]
enable_plugin devstack-plugin-container https://git.openstack.org/openstack/devstack-plugin-container
enable_service etcd3
enable_service container
enable_service k8s-master
# kubeadm token generate
K8S_TOKEN="9agf12.zsu5uh2m4pzt3qba"
...
END
And using the following config in worker node:
.. code-block:: ini
cat > /opt/stack/devstack/local.conf << END
[[local|localrc]]
SERVICE_HOST=10.0.0.11 # change this to controller's IP address
enable_plugin devstack-plugin-container https://git.openstack.org/openstack/devstack-plugin-container
enable_service container
enable_service k8s-node
# kubeadm token generate
K8S_TOKEN="9agf12.zsu5uh2m4pzt3qba"
...
END
3. Run devstack
--------------------

154
devstack/lib/k8s Normal file
View File

@ -0,0 +1,154 @@
#!/bin/bash
# Dependencies:
#
# - functions
# - ``STACK_USER`` must be defined
# stack.sh
# --------
# - install_k8s
# The following variables are assumed to be defined by certain functions:
#
# - ``http_proxy`` ``https_proxy`` ``no_proxy``
# Save trace setting
_XTRACE_DOCKER=$(set +o | grep xtrace)
set +o xtrace
# Defaults
# --------
K8S_TOKEN=${K8S_TOKEN:-""}
K8S_API_SERVER_IP=${K8S_API_SERVER_IP:-$SERVICE_HOST}
K8S_NODE_IP=${K8S_NODE_IP:-$HOST_IP}
K8S_API_SERVER_PORT=${K8S_API_SERVER_PORT:-6443}
K8S_POD_NETWORK_CIDR=${K8S_POD_NETWORK_CIDR:-10.244.0.0/16}
K8S_SERVICE_NETWORK_CIDR=${K8S_SERVICE_NETWORK_CIDR:-10.96.0.0/12}
K8S_VERSION=${K8S_VERSION:-1.14.1-00}
K8S_NETWORK_ADDON=${K8S_NETWORK_ADDON:-flannel}
# Functions
# ---------
function is_k8s_enabled {
[[ ,${ENABLED_SERVICES} =~ ,"k8s-" ]] && return 0
return 1
}
function install_kubeadm {
if is_ubuntu; then
apt_get install apt-transport-https curl
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
sudo add-apt-repository -y \
"deb https://apt.kubernetes.io/ kubernetes-xenial main"
REPOS_UPDATED=False apt_get_update
apt_get install kubelet=$K8S_VERSION kubeadm=$K8S_VERSION kubectl=$K8S_VERSION
sudo apt-mark hold kubelet kubeadm kubectl
# NOTE(hongbin): This work-around an issue that kubelet pick a wrong
# IP address if the node has multiple network interfaces.
# See https://github.com/kubernetes/kubeadm/issues/203
echo "KUBELET_EXTRA_ARGS=--node-ip=$K8S_NODE_IP" | sudo tee -a /etc/default/kubelet
sudo systemctl daemon-reload && sudo systemctl restart kubelet
else
(>&2 echo "WARNING: kubeadm installation is not supported in this distribution.")
fi
}
function kubeadm_init {
local kubeadm_config_file=$(mktemp)
cat <<EOF | sudo tee $kubeadm_config_file >/dev/null
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
etcd:
external:
endpoints:
- "http://${SERVICE_HOST}:${ETCD_PORT}"
networking:
podSubnet: "${K8S_POD_NETWORK_CIDR}"
serviceSubnet: "${K8S_SERVICE_NETWORK_CIDR}"
---
apiVersion: kubeadm.k8s.io/v1beta1
bootstrapTokens:
- token: "${K8S_TOKEN}"
ttl: 0s
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: "${K8S_API_SERVER_IP}"
bindPort: ${K8S_API_SERVER_PORT}
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
failSwapOn: false
EOF
sudo kubeadm init --config $kubeadm_config_file --ignore-preflight-errors Swap
local kube_config_file=$HOME/.kube/config
sudo mkdir -p $(dirname ${kube_config_file})
sudo cp /etc/kubernetes/admin.conf $kube_config_file
safe_chown $STACK_USER:$STACK_USER $kube_config_file
if [[ "$K8S_NETWORK_ADDON" == "flannel" ]]; then
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/a70459be0084506e4ec919aa1c114638878db11b/Documentation/kube-flannel.yml
fi
}
function kubeadm_join {
local kubeadm_config_file=$(mktemp)
cat <<EOF | sudo tee $kubeadm_config_file >/dev/null
apiVersion: kubeadm.k8s.io/v1beta1
kind: JoinConfiguration
discovery:
bootstrapToken:
apiServerEndpoint: "${K8S_API_SERVER_IP}:${K8S_API_SERVER_PORT}"
token: "${K8S_TOKEN}"
unsafeSkipCAVerification: true
tlsBootstrapToken: "${K8S_TOKEN}"
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
failSwapOn: false
EOF
sudo kubeadm join --config $kubeadm_config_file --ignore-preflight-errors Swap
}
function start_collect_logs {
wait_for_kube_service 180 component=kube-controller-manager
wait_for_kube_service 60 component=kube-apiserver
wait_for_kube_service 30 component=kube-scheduler
wait_for_kube_service 30 k8s-app=kube-proxy
run_process kube-controller-manager "/usr/bin/kubectl logs -n kube-system -f -l component=kube-controller-manager"
run_process kube-apiserver "/usr/bin/kubectl logs -n kube-system -f -l component=kube-apiserver"
run_process kube-scheduler "/usr/bin/kubectl logs -n kube-system -f -l component=kube-scheduler"
run_process kube-proxy "/usr/bin/kubectl logs -n kube-system -f -l k8s-app=kube-proxy"
}
function wait_for_kube_service {
local timeout=$1
local selector=$2
local rval=0
time_start "wait_for_service"
timeout $timeout bash -x <<EOF || rval=$?
NAME=""
while [[ "\$NAME" == "" ]]; do
sleep 1
NAME=\$(kubectl wait --for=condition=Ready pod -n kube-system -l $selector -o name)
done
EOF
time_stop "wait_for_service"
# Figure out what's happening on platforms where this doesn't work
if [[ "$rval" != 0 ]]; then
echo "Didn't find kube service after $timeout seconds"
kubectl get pods -n kube-system -l $selector
fi
return $rval
}
function kubeadm_reset {
sudo kubeadm reset --force
}
# Restore xtrace
$_XTRACE_DOCKER

View File

@ -7,6 +7,7 @@ set -o xtrace
echo_summary "container's plugin.sh was called..."
source $DEST/devstack-plugin-container/devstack/lib/docker
source $DEST/devstack-plugin-container/devstack/lib/crio
source $DEST/devstack-plugin-container/devstack/lib/k8s
(set -o posix; set)
if is_service_enabled container; then
@ -40,5 +41,30 @@ if is_service_enabled container; then
fi
fi
if is_k8s_enabled; then
if [[ "$1" == "stack" && "$2" == "install" ]]; then
install_kubeadm
elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
if is_service_enabled k8s-master; then
kubeadm_init
elif is_service_enabled k8s-node; then
kubeadm_join
fi
elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
if is_service_enabled k8s-master; then
start_collect_logs
fi
fi
if [[ "$1" == "unstack" ]]; then
kubeadm_reset
fi
if [[ "$1" == "clean" ]]; then
# nothing needed here
:
fi
fi
# Restore xtrace
$XTRACE

View File

@ -8,3 +8,11 @@ ENABLE_IPV6=${ENABLE_IPV6:-false}
# Enable container services
enable_service container
# Enable k8s services
if [[ ,${ENABLED_SERVICES} =~ ,"k8s-master" ]]; then
enable_service kube-controller-manager
enable_service kube-apiserver
enable_service kube-scheduler
enable_service kube-proxy
fi