Add devstack plugin for stackube
The plugin install components including * OpenStack Keystone, Neutron and Cinder with Ceph backend * Kubernetes with frakti container runtime * Kubernetes auth via keystone Implements: blueprint kubernetes-deploy Change-Id: Ib2ad60b71bc51bdedaefd7014c83ed7ba56414e3 Signed-off-by: Pengfei Ni <feiskyer@gmail.com>
This commit is contained in:
Pengfei Ni
parent
d26f5b2692
commit
291127f0d1
50
devstack/README.md
Normal file
50
devstack/README.md
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
# devstack plugin
|
||||||
|
|
||||||
|
devstack plugin for stackube.
|
||||||
|
|
||||||
|
## All-in-one
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# create stack user
|
||||||
|
sudo useradd -s /bin/bash -d /opt/stack -m stack
|
||||||
|
echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack
|
||||||
|
sudo su - stack
|
||||||
|
|
||||||
|
git clone https://git.openstack.org/openstack-dev/devstack -b stable/ocata
|
||||||
|
cd devstack
|
||||||
|
```
|
||||||
|
|
||||||
|
Create `local.conf` from [local.conf.sample](local.conf.sample) and then run `./stack.sh` to install.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./stack.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Wait a while for installation compelete, then setup kubernetes and OpenStack client:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Kubernetes
|
||||||
|
export KUBECONFIG=$HOME/admin.conf
|
||||||
|
kubectl cluster-info
|
||||||
|
|
||||||
|
# OpenStack
|
||||||
|
source openrc admin admin
|
||||||
|
openstack service list
|
||||||
|
```
|
||||||
|
|
||||||
|
## Add a node
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# create stack user
|
||||||
|
sudo useradd -s /bin/bash -d /opt/stack -m stack
|
||||||
|
echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack
|
||||||
|
sudo su - stack
|
||||||
|
|
||||||
|
git clone https://git.openstack.org/openstack-dev/devstack -b stable/ocata
|
||||||
|
cd devstack
|
||||||
|
```
|
||||||
|
|
||||||
|
Create `local.conf` from [local.conf.node.sample](local.conf.node.sample), set `HOST_IP` to local host's IP, set `SERVICE_HOST` to master's IP and set `KUBEADM_TOKEN` to kubeadm token (could be got by `kubeadm token list`).
|
||||||
|
|
||||||
|
Then run `./stack.sh` to install.
|
||||||
|
|
||||||
13
devstack/kubeadm.yaml
Normal file
13
devstack/kubeadm.yaml
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
kind: MasterConfiguration
|
||||||
|
apiVersion: kubeadm.k8s.io/v1alpha1
|
||||||
|
kubernetesVersion: "stable"
|
||||||
|
controllerManagerExtraArgs:
|
||||||
|
horizontal-pod-autoscaler-use-rest-clients: "true"
|
||||||
|
horizontal-pod-autoscaler-sync-period: "10s"
|
||||||
|
node-monitor-grace-period: "10s"
|
||||||
|
feature-gates: "AllAlpha=true"
|
||||||
|
enable-dynamic-provisioning: "true"
|
||||||
|
apiServerExtraArgs:
|
||||||
|
runtime-config: "api/all=true"
|
||||||
|
feature-gates: "AllAlpha=true"
|
||||||
|
experimental-keystone-url: "https://KEYSTONE_HOST:5000/v2.0"
|
||||||
29
devstack/local.conf.node.sample
Normal file
29
devstack/local.conf.node.sample
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
[[local|localrc]]
|
||||||
|
# Set to local host's IP
|
||||||
|
HOST_IP=10.0.0.3
|
||||||
|
# Set to master node's IP
|
||||||
|
SERVICE_HOST="10.0.0.2"
|
||||||
|
MYSQL_HOST=${SERVICE_HOST}
|
||||||
|
RABBIT_HOST=${SERVICE_HOST}
|
||||||
|
ADMIN_PASSWORD=password
|
||||||
|
DATABASE_PASSWORD=$ADMIN_PASSWORD
|
||||||
|
RABBIT_PASSWORD=$ADMIN_PASSWORD
|
||||||
|
SERVICE_PASSWORD=$ADMIN_PASSWORD
|
||||||
|
|
||||||
|
ENABLED_SERVICES=""
|
||||||
|
|
||||||
|
# Neutron
|
||||||
|
enable_service q-agt
|
||||||
|
|
||||||
|
# Kubernetes
|
||||||
|
STACKUBE_HOME=${STACKUBE_HOME:-$DEST/stackube}
|
||||||
|
# Kubernetes master IP, should set to master IP for minions
|
||||||
|
KUBERNETES_MASTER_IP=${SERVICE_HOST}
|
||||||
|
# Get token by running following command on the master node
|
||||||
|
# sudo kubeadm token list | awk '/kubeadm/{print $1}'
|
||||||
|
KUBEADM_TOKEN="token"
|
||||||
|
CONTAINER_CIDR="10.244.2.0/24"
|
||||||
|
enable_plugin stackube git://git.openstack.org/openstack/stackube
|
||||||
|
enable_service stackube
|
||||||
|
# Enable kubernetes node components
|
||||||
|
enable_service kubernetes_node
|
||||||
33
devstack/local.conf.sample
Normal file
33
devstack/local.conf.sample
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
[[local|localrc]]
|
||||||
|
ADMIN_PASSWORD=password
|
||||||
|
DATABASE_PASSWORD=$ADMIN_PASSWORD
|
||||||
|
RABBIT_PASSWORD=$ADMIN_PASSWORD
|
||||||
|
SERVICE_PASSWORD=$ADMIN_PASSWORD
|
||||||
|
|
||||||
|
ENABLED_SERVICES="tls-proxy"
|
||||||
|
|
||||||
|
# Keystone
|
||||||
|
enable_service key
|
||||||
|
enable_service mysql
|
||||||
|
enable_service rabbit
|
||||||
|
|
||||||
|
# Neutron
|
||||||
|
enable_service neutron
|
||||||
|
enable_service q-agt
|
||||||
|
enable_service q-dhcp
|
||||||
|
enable_service q-l3
|
||||||
|
enable_service q-svc
|
||||||
|
enable_plugin neutron-lbaas git://git.openstack.org/openstack/neutron-lbaas stable/ocata
|
||||||
|
enable_service q-lbaasv2
|
||||||
|
NEUTRON_LBAAS_SERVICE_PROVIDERV2="LOADBALANCERV2:Haproxy:neutron_lbaas.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default"
|
||||||
|
|
||||||
|
# Kubernetes
|
||||||
|
enable_plugin stackube git://git.openstack.org/openstack/stackube
|
||||||
|
enable_service stackube
|
||||||
|
# Enable kubernetes master components
|
||||||
|
enable_service kubernetes_master
|
||||||
|
|
||||||
|
# Cinder
|
||||||
|
ENABLE_CEPH_CINDER=True
|
||||||
|
enable_service cinder c-api c-vol c-sch c-bak
|
||||||
|
enable_plugin devstack-plugin-ceph git://git.openstack.org/openstack/devstack-plugin-ceph
|
||||||
215
devstack/plugin.sh
Normal file
215
devstack/plugin.sh
Normal file
@ -0,0 +1,215 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
STACKUBE_ROOT=$(dirname "${BASH_SOURCE}")
|
||||||
|
|
||||||
|
function configure_cni {
|
||||||
|
sudo mkdir -p /etc/cni/net.d
|
||||||
|
sudo sh -c "cat >/etc/cni/net.d/10-mynet.conf <<EOF
|
||||||
|
{
|
||||||
|
\"cniVersion\": \"0.3.0\",
|
||||||
|
\"name\": \"mynet\",
|
||||||
|
\"type\": \"bridge\",
|
||||||
|
\"bridge\": \"cni0\",
|
||||||
|
\"isGateway\": true,
|
||||||
|
\"ipMasq\": true,
|
||||||
|
\"ipam\": {
|
||||||
|
\"type\": \"host-local\",
|
||||||
|
\"subnet\": \"${CONTAINER_CIDR}\",
|
||||||
|
\"routes\": [
|
||||||
|
{ \"dst\": \"0.0.0.0/0\" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EOF"
|
||||||
|
sudo sh -c 'cat >/etc/cni/net.d/99-loopback.conf <<EOF
|
||||||
|
{
|
||||||
|
"cniVersion": "0.3.0",
|
||||||
|
"type": "loopback"
|
||||||
|
}
|
||||||
|
EOF'
|
||||||
|
}
|
||||||
|
|
||||||
|
function install_docker {
|
||||||
|
if is_ubuntu; then
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y docker.io
|
||||||
|
elif is_fedora; then
|
||||||
|
sudo yum install -y docker
|
||||||
|
else
|
||||||
|
exit_distro_not_supported
|
||||||
|
fi
|
||||||
|
|
||||||
|
sudo systemctl start docker
|
||||||
|
}
|
||||||
|
|
||||||
|
function install_hyper {
|
||||||
|
if is_ubuntu; then
|
||||||
|
sudo apt-get update && sudo apt-get install -y qemu libvirt-bin
|
||||||
|
elif is_fedora; then
|
||||||
|
sudo yum install -y libvirt
|
||||||
|
fi
|
||||||
|
|
||||||
|
sudo systemctl restart libvirtd
|
||||||
|
|
||||||
|
if command -v /usr/bin/hyperd > /dev/null 2>&1; then
|
||||||
|
echo "hyperd already installed on this host, using it instead"
|
||||||
|
else
|
||||||
|
curl -sSL https://hypercontainer.io/install | sudo bash
|
||||||
|
fi
|
||||||
|
sudo sh -c 'cat>/etc/hyper/config <<EOF
|
||||||
|
Kernel=/var/lib/hyper/kernel
|
||||||
|
Initrd=/var/lib/hyper/hyper-initrd.img
|
||||||
|
Hypervisor=qemu
|
||||||
|
StorageDriver=overlay
|
||||||
|
gRPCHost=127.0.0.1:22318
|
||||||
|
EOF'
|
||||||
|
}
|
||||||
|
|
||||||
|
function install_frakti {
|
||||||
|
if command -v /usr/bin/frakti > /dev/null 2>&1; then
|
||||||
|
sudo rm -f /usr/bin/frakti
|
||||||
|
fi
|
||||||
|
sudo curl -sSL https://github.com/kubernetes/frakti/releases/download/${FRAKTI_VERSION}/frakti -o /usr/bin/frakti
|
||||||
|
sudo chmod +x /usr/bin/frakti
|
||||||
|
cgroup_driver=$(sudo docker info | awk '/Cgroup Driver/{print $3}')
|
||||||
|
sudo sh -c "cat > /lib/systemd/system/frakti.service <<EOF
|
||||||
|
[Unit]
|
||||||
|
Description=Hypervisor-based container runtime for Kubernetes
|
||||||
|
Documentation=https://github.com/kubernetes/frakti
|
||||||
|
After=network.target
|
||||||
|
[Service]
|
||||||
|
ExecStart=/usr/bin/frakti --v=3 \
|
||||||
|
--log-dir=/var/log/frakti \
|
||||||
|
--logtostderr=false \
|
||||||
|
--cgroup-driver=${cgroup_driver} \
|
||||||
|
--listen=/var/run/frakti.sock \
|
||||||
|
--streaming-server-addr=%H \
|
||||||
|
--hyper-endpoint=127.0.0.1:22318
|
||||||
|
MountFlags=shared
|
||||||
|
TasksMax=8192
|
||||||
|
LimitNOFILE=1048576
|
||||||
|
LimitNPROC=1048576
|
||||||
|
LimitCORE=infinity
|
||||||
|
TimeoutStartSec=0
|
||||||
|
Restart=on-abnormal
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
EOF"
|
||||||
|
}
|
||||||
|
|
||||||
|
function install_kubelet {
|
||||||
|
if is_fedora; then
|
||||||
|
sudo sh -c 'cat > /etc/yum.repos.d/kubernetes.repo <<EOF
|
||||||
|
[kubernetes]
|
||||||
|
name=Kubernetes
|
||||||
|
baseurl=http://yum.kubernetes.io/repos/kubernetes-el7-x86_64
|
||||||
|
enabled=1
|
||||||
|
gpgcheck=1
|
||||||
|
repo_gpgcheck=1
|
||||||
|
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
|
||||||
|
https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
|
||||||
|
EOF'
|
||||||
|
sudo setenforce 0
|
||||||
|
sudo sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
|
||||||
|
sudo yum install -y kubernetes-cni kubelet kubeadm kubectl
|
||||||
|
elif is_ubuntu; then
|
||||||
|
sudo apt-get update && sudo apt-get install -y apt-transport-https
|
||||||
|
sudo curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
|
||||||
|
sudo sh -c 'cat > /etc/apt/sources.list.d/kubernetes.list <<EOF
|
||||||
|
deb http://apt.kubernetes.io/ kubernetes-xenial main
|
||||||
|
EOF'
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y kubernetes-cni kubelet kubeadm kubectl
|
||||||
|
else
|
||||||
|
exit_distro_not_supported
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function install_master {
|
||||||
|
sed -i "s/KEYSTONE_HOST/${SERVICE_HOST}/g" ${STACKUBE_ROOT}/kubeadm.yaml
|
||||||
|
sudo kubeadm init kubeadm init --pod-network-cidr ${CLUSTER_CIDR} --config ${STACKUBE_ROOT}/kubeadm.yaml
|
||||||
|
# Enable schedule pods on the master for testing.
|
||||||
|
sudo cp /etc/kubernetes/admin.conf $HOME/
|
||||||
|
sudo chown $(id -u):$(id -g) $HOME/admin.conf
|
||||||
|
export KUBECONFIG=$HOME/admin.conf
|
||||||
|
kubectl taint nodes --all node-role.kubernetes.io/master-
|
||||||
|
}
|
||||||
|
|
||||||
|
function install_node {
|
||||||
|
if [ "${KUBEADM_TOKEN}" = "" ]; then
|
||||||
|
echo "KUBEADM_TOKEN must be set for node"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sudo kubeadm join --token "${KUBEADM_TOKEN}" ${KUBERNETES_MASTER_IP}:${KUBERNETES_MASTER_PORT}
|
||||||
|
}
|
||||||
|
|
||||||
|
function configure_kubelet {
|
||||||
|
sudo sed -i '2 i\Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --container-runtime-endpoint=/var/run/frakti.sock --feature-gates=AllAlpha=true"' /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
}
|
||||||
|
|
||||||
|
function remove_kubernetes {
|
||||||
|
sudo kubeadm reset
|
||||||
|
sudo systemctl stop kubelet
|
||||||
|
|
||||||
|
if is_fedora; then
|
||||||
|
sudo yum remove -y qemu-hyper hyperstart hyper-container libvirt
|
||||||
|
sudo yum remove -y kubernetes-cni kubelet kubeadm kubectl docker
|
||||||
|
elif is_ubuntu; then
|
||||||
|
sudo apt-get remove -y hyperstart hyper-container qemu libvirt-bin
|
||||||
|
sudo apt-get remove -y kubernetes-cni kubelet kubeadm kubectl docker
|
||||||
|
fi
|
||||||
|
|
||||||
|
sudo rm -rf /usr/bin/frakti /etc/cni/net.d /lib/systemd/system/frakti.service
|
||||||
|
}
|
||||||
|
|
||||||
|
function install_stackube {
|
||||||
|
install_docker
|
||||||
|
install_hyper
|
||||||
|
install_frakti
|
||||||
|
install_kubelet
|
||||||
|
}
|
||||||
|
|
||||||
|
function init_stackube {
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl restart docker
|
||||||
|
sudo systemctl restart libvirtd
|
||||||
|
sudo systemctl restart hyperd
|
||||||
|
sudo systemctl restart frakti
|
||||||
|
|
||||||
|
if is_service_enabled kubernetes_master; then
|
||||||
|
install_master
|
||||||
|
elif is_service_enabled kubernetes_node; then
|
||||||
|
install_node
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function configure_stackube {
|
||||||
|
configure_cni
|
||||||
|
configure_kubelet
|
||||||
|
}
|
||||||
|
|
||||||
|
# check for service enabled
|
||||||
|
if is_service_enabled stackube; then
|
||||||
|
|
||||||
|
if [[ "$1" == "stack" && "$2" == "install" ]]; then
|
||||||
|
echo_summary "Installing stackube"
|
||||||
|
install_stackube
|
||||||
|
|
||||||
|
elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
|
||||||
|
echo_summary "Configuring stackube"
|
||||||
|
configure_stackube
|
||||||
|
|
||||||
|
elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
|
||||||
|
# Initialize and start the stackube service
|
||||||
|
echo_summary "Initializing stackube"
|
||||||
|
init_stackube
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$1" == "unstack" ]]; then
|
||||||
|
remove_kubernetes
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$1" == "clean" ]]; then
|
||||||
|
echo ''
|
||||||
|
fi
|
||||||
|
fi
|
||||||
15
devstack/settings
Normal file
15
devstack/settings
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
STACKUBE_HOME=${STACKUBE_HOME:-$DEST/stackube}
|
||||||
|
# Kubernetes master IP, should set to master IP for minions
|
||||||
|
KUBERNETES_MASTER_IP=${KUBERNETES_MASTER_IP:-${SERVICE_HOST}}
|
||||||
|
KUBERNETES_MASTER_PORT=${KUBERNETES_MASTER_PORT:-6443}
|
||||||
|
# Kubeadm token, only for minions
|
||||||
|
KUBEADM_TOKEN=${KUBEADM_TOKEN:-""}
|
||||||
|
# The version of frakti
|
||||||
|
FRAKTI_VERSION=${FRAKTI_VERSION:-"v0.2"}
|
||||||
|
# Kubernetes CLUSTER CIDR
|
||||||
|
CLUSTER_CIDR=${CLUSTER_CIDR:-"10.244.0.0/16"}
|
||||||
|
# CNI Configurations
|
||||||
|
# Brige network plugin is used now
|
||||||
|
# TODO: replace network plugin with stackube-cni
|
||||||
|
CONTAINER_CIDR=${CONTAINER_CIDR:-"10.244.1.0/24"}
|
||||||
|
CNI_VERSION=${CNI_VERSION:-"v0.5.2"}
|
||||||
Loading…
Reference in New Issue
Block a user