diff --git a/devstack/plugin.sh b/devstack/plugin.sh index ea1f0c7a..96fa6e30 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -21,7 +21,7 @@ function install_qinlingclient { function install_k8s { pushd $QINLING_DIR - source tools/gate/setup_gate.sh + source tools/gate/kubeadm/setup_gate.sh popd # Pre-fetch the default docker image for python runtime and image function diff --git a/tools/gate/kubeadm/funcs/common.sh b/tools/gate/kubeadm/funcs/common.sh new file mode 100644 index 00000000..8f359c96 --- /dev/null +++ b/tools/gate/kubeadm/funcs/common.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function base_install { + if [ "x$HOST_OS" == "xubuntu" ]; then + sudo apt-get update -y + sudo apt-get install -y --no-install-recommends \ + iproute2 \ + iptables \ + ipcalc \ + nmap \ + lshw \ + screen + elif [ "x$HOST_OS" == "xcentos" ]; then + sudo yum install -y \ + epel-release + # ipcalc is in the initscripts package + sudo yum install -y \ + iproute \ + iptables \ + initscripts \ + nmap \ + lshw + elif [ "x$HOST_OS" == "xfedora" ]; then + sudo dnf install -y \ + iproute \ + iptables \ + ipcalc \ + nmap \ + lshw + fi +} + +function gate_base_setup { + # Install base requirements + base_install +} + +function create_k8s_screen { + # Starts a proxy to the Kubernetes API server in a screen session + sudo screen -S kube_proxy -X quit || true + sudo screen -dmS kube_proxy && sudo screen -S kube_proxy -X screen -t kube_proxy + sudo screen -S kube_proxy -p kube_proxy -X stuff 'kubectl proxy --accept-hosts=".*" --address="0.0.0.0"\n' +} \ No newline at end of file diff --git a/tools/gate/kubeadm/funcs/network.sh b/tools/gate/kubeadm/funcs/network.sh new file mode 100644 index 00000000..9f22a5d7 --- /dev/null +++ b/tools/gate/kubeadm/funcs/network.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function net_default_iface { + sudo ip -4 route list 0/0 | awk '{ print $5; exit }' +} + +function net_default_host_addr { + sudo ip addr | awk "/inet / && /$(net_default_iface)/{print \$2; exit }" +} + +function net_default_host_ip { + echo $(net_default_host_addr) | awk -F '/' '{ print $1; exit }' +} + +function net_resolv_pre_kube { + sudo cp -f /etc/resolv.conf /etc/resolv-pre-kube.conf + sudo rm -f /etc/resolv.conf + cat << EOF | sudo tee /etc/resolv.conf +nameserver ${UPSTREAM_DNS} +EOF +} + +function net_resolv_post_kube { + sudo cp -f /etc/resolv-pre-kube.conf /etc/resolv.conf +} + +function net_hosts_pre_kube { + sudo cp -f /etc/hosts /etc/hosts-pre-kube + sudo sed -i "/$(hostname)/d" /etc/hosts + sudo sed -i "/127.0.0.1/d" /etc/hosts + sudo sed -i "1 i 127.0.0.1 localhost" /etc/hosts + + host_ip=$(net_default_host_ip) + echo "${host_ip} $(hostname)" | sudo tee -a /etc/hosts +} + +function net_hosts_post_kube { + sudo cp -f /etc/hosts-pre-kube /etc/hosts +} \ No newline at end of file diff --git a/tools/gate/kubeadm/playbook/deploy_k8s.yaml b/tools/gate/kubeadm/playbook/deploy_k8s.yaml new file mode 100644 index 00000000..3bb37120 --- /dev/null +++ b/tools/gate/kubeadm/playbook/deploy_k8s.yaml @@ -0,0 +1,25 @@ +--- +- name: Deploy k8s cluster + hosts: localhost + become: true + become_method: sudo + gather_facts: false + + pre_tasks: + - name: install python 2 + raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) + + - name: Gather facts after python installation + setup: filter=ansible_* + + - name: Install packages + apt: name={{ item }} state=present update_cache=yes + with_items: + - ansible + - unzip + - httpie + + roles: + - docker + - k8s_cli + - kube_master \ No newline at end of file diff --git a/tools/gate/kubeadm/playbook/roles/docker/defaults/main.yml b/tools/gate/kubeadm/playbook/roles/docker/defaults/main.yml new file mode 100644 index 00000000..7367bf20 --- /dev/null +++ b/tools/gate/kubeadm/playbook/roles/docker/defaults/main.yml @@ -0,0 +1,2 @@ +--- +docker_version: 17.03.2~ce-0~ubuntu-xenial \ No newline at end of file diff --git a/tools/gate/kubeadm/playbook/roles/docker/tasks/main.yml b/tools/gate/kubeadm/playbook/roles/docker/tasks/main.yml new file mode 100644 index 00000000..ecdb8aa7 --- /dev/null +++ b/tools/gate/kubeadm/playbook/roles/docker/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Install packages + apt: name={{ item }} state=present update_cache=yes + with_items: + - apt-transport-https + - ca-certificates + - curl + +- name: Add Docker’s official GPG key + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + +- name: Set up the stable repository + apt_repository: repo='deb https://download.docker.com/linux/ubuntu xenial stable' state=present + +- name: Install docker-ce + apt: + name: docker-ce={{ docker_version }} + state: present + update_cache: yes \ No newline at end of file diff --git a/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/install_kube_prompt_tasks.yml b/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/install_kube_prompt_tasks.yml new file mode 100644 index 00000000..ec093441 --- /dev/null +++ b/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/install_kube_prompt_tasks.yml @@ -0,0 +1,26 @@ +--- +- name: Create temporary directory + file: + state: directory + path: ~/kube-prompt + register: dir + +- name: Download kube-prompt + get_url: + url: https://github.com/c-bata/kube-prompt/releases/download/v1.0.3/kube-prompt_v1.0.3_linux_amd64.zip + dest: "{{ dir.path }}/kube-prompt_v1.0.3_linux_amd64.zip" + force: no + +- name: Unarchive kube-prompt + unarchive: + src: "{{ dir.path }}/kube-prompt_v1.0.3_linux_amd64.zip" + dest: "{{ dir.path }}" + remote_src: yes + creates: "{{ dir.path }}/kube-prompt" + +- name: Copy kube-prompt to bin + copy: + dest: /usr/local/bin/kprompt + src: "{{ dir.path }}/kube-prompt" + mode: u+x + remote_src: yes \ No newline at end of file diff --git a/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/main.yml b/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/main.yml new file mode 100644 index 00000000..a5884442 --- /dev/null +++ b/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Add Kubernetes GPG key + apt_key: + url: https://packages.cloud.google.com/apt/doc/apt-key.gpg + state: present + +- name: Set up Kubernetes repository + apt_repository: repo='deb http://apt.kubernetes.io/ kubernetes-xenial main' state=present + +- name: Install kube CLIs + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - kubelet + - kubeadm + - kubectl + +- import_tasks: install_kube_prompt_tasks.yml \ No newline at end of file diff --git a/tools/gate/kubeadm/playbook/roles/kube_master/tasks/main.yml b/tools/gate/kubeadm/playbook/roles/kube_master/tasks/main.yml new file mode 100644 index 00000000..3637a731 --- /dev/null +++ b/tools/gate/kubeadm/playbook/roles/kube_master/tasks/main.yml @@ -0,0 +1,46 @@ +--- +- name: disable swap + shell: swapoff -a + +- name: drain the node if needed + shell: kubectl drain {{ ansible_nodename }} --delete-local-data --force --ignore-daemonsets + ignore_errors: True + +- name: delete node if needed + shell: kubectl delete node {{ ansible_nodename }} + ignore_errors: True + +- name: reset kubeadm setup + shell: kubeadm reset + ignore_errors: True + +- name: Init master + shell: kubeadm init --pod-network-cidr=192.168.0.0/16 --kubernetes-version=1.9.3 + register: kubeadm_init + +- fail: + msg: "kubeadm init command failed." + when: kubeadm_init.stdout.find ("kubeadm join") == -1 + +- name: Store kubeadm join string + set_fact: + join_str: "{{ kubeadm_init.stdout | regex_search('kubeadm join(.*)$') }}" + +- name: Prepare kube config + shell: mkdir -p {{ item }}/.kube && cp -a /etc/kubernetes/admin.conf {{ item }}/.kube/config && chmod 755 {{ item }}/.kube/config + with_items: + - "{{ ansible_env.HOME }}" + - /home/ubuntu + - /opt/stack/new + +- name: Allow pod on master + shell: kubectl taint nodes --all node-role.kubernetes.io/master- + +- name: Download calico yaml + get_url: + url: https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml + dest: "{{ ansible_env.HOME }}/calico.yaml" + force: no + +- name: Install network plugin + shell: kubectl apply -f {{ ansible_env.HOME }}/calico.yaml \ No newline at end of file diff --git a/tools/gate/kubeadm/setup_gate.sh b/tools/gate/kubeadm/setup_gate.sh new file mode 100755 index 00000000..eafa6e47 --- /dev/null +++ b/tools/gate/kubeadm/setup_gate.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +export WORK_DIR=$(pwd) +source ${WORK_DIR}/tools/gate/kubeadm/vars.sh +source ${WORK_DIR}/tools/gate/kubeadm/funcs/common.sh +source ${WORK_DIR}/tools/gate/kubeadm/funcs/network.sh + +# Do the basic node setup for running the gate +gate_base_setup +net_resolv_pre_kube +net_hosts_pre_kube + +# Setup the K8s Cluster +sudo apt-add-repository --yes ppa:ansible/ansible && sudo apt-get update -y -qq && sudo apt-get install ansible -y -qq +ansible-playbook ${WORK_DIR}/tools/gate/kubeadm/playbook/deploy_k8s.yaml + +# waits until kubectl can access the api server +sudo chown $(id -u):$(id -g) ${HOME}/.kube/config +end=$(($(date +%s) + 600)) +READY="False" +while true; do + READY=$(kubectl get nodes --no-headers=true | awk "{ print \$2 }" | head -1) + [ "$READY" == "Ready" ] && break || true + sleep 2 + now=$(date +%s) + [ $now -gt $end ] && echo "Failed to setup kubernetes cluster in time" && exit -1 +done + +create_k8s_screen + +#net_hosts_post_kube +#net_resolv_post_kube diff --git a/tools/gate/kubeadm/vars.sh b/tools/gate/kubeadm/vars.sh new file mode 100644 index 00000000..94e27ce7 --- /dev/null +++ b/tools/gate/kubeadm/vars.sh @@ -0,0 +1,19 @@ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set work dir if not already done +: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"} + +# Get Host OS +source /etc/os-release +export HOST_OS=${HOST_OS:="${ID}"} + +# Set versions of K8s to use +export KUBE_VERSION=${KUBE_VERSION:-"v1.8.0"} + +# Set Upstream DNS +export UPSTREAM_DNS=${UPSTREAM_DNS:-"8.8.8.8"} \ No newline at end of file