Use kubeadm for devstack

Using kubeadm instead of other installing tools gives us more control of our
own devstack installation.

Change-Id: I0bffcc091fea70ea62952b21fb325c2cdde26b86

devstack/plugin.sh

@@ -21,7 +21,7 @@ function install_qinlingclient {
 
 function install_k8s {
     pushd $QINLING_DIR
-    source tools/gate/setup_gate.sh
+    source tools/gate/kubeadm/setup_gate.sh
     popd
 
     # Pre-fetch the default docker image for python runtime and image function

tools/gate/kubeadm/funcs/common.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function base_install {
+  if [ "x$HOST_OS" == "xubuntu" ]; then
+    sudo apt-get update -y
+    sudo apt-get install -y --no-install-recommends \
+      iproute2 \
+      iptables \
+      ipcalc \
+      nmap \
+      lshw \
+      screen
+  elif [ "x$HOST_OS" == "xcentos" ]; then
+    sudo yum install -y \
+      epel-release
+    # ipcalc is in the initscripts package
+    sudo yum install -y \
+      iproute \
+      iptables \
+      initscripts \
+      nmap \
+      lshw
+  elif [ "x$HOST_OS" == "xfedora" ]; then
+    sudo dnf install -y \
+      iproute \
+      iptables \
+      ipcalc \
+      nmap \
+      lshw
+  fi
+}
+
+function gate_base_setup {
+  # Install base requirements
+  base_install
+}
+
+function create_k8s_screen {
+  # Starts a proxy to the Kubernetes API server in a screen session
+  sudo screen -S kube_proxy -X quit || true
+  sudo screen -dmS kube_proxy && sudo screen -S kube_proxy -X screen -t kube_proxy
+  sudo screen -S kube_proxy -p kube_proxy -X stuff 'kubectl proxy --accept-hosts=".*" --address="0.0.0.0"\n'
+}

tools/gate/kubeadm/funcs/network.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function net_default_iface {
+ sudo ip -4 route list 0/0 | awk '{ print $5; exit }'
+}
+
+function net_default_host_addr {
+ sudo ip addr | awk "/inet / && /$(net_default_iface)/{print \$2; exit }"
+}
+
+function net_default_host_ip {
+ echo $(net_default_host_addr) | awk -F '/' '{ print $1; exit }'
+}
+
+function net_resolv_pre_kube {
+  sudo cp -f /etc/resolv.conf /etc/resolv-pre-kube.conf
+  sudo rm -f /etc/resolv.conf
+  cat << EOF | sudo tee /etc/resolv.conf
+nameserver ${UPSTREAM_DNS}
+EOF
+}
+
+function net_resolv_post_kube {
+  sudo cp -f /etc/resolv-pre-kube.conf /etc/resolv.conf
+}
+
+function net_hosts_pre_kube {
+  sudo cp -f /etc/hosts /etc/hosts-pre-kube
+  sudo sed -i "/$(hostname)/d" /etc/hosts
+  sudo sed -i "/127.0.0.1/d" /etc/hosts
+  sudo sed -i "1 i 127.0.0.1 localhost" /etc/hosts
+
+  host_ip=$(net_default_host_ip)
+  echo "${host_ip} $(hostname)" | sudo tee -a /etc/hosts
+}
+
+function net_hosts_post_kube {
+  sudo cp -f /etc/hosts-pre-kube /etc/hosts
+}

tools/gate/kubeadm/playbook/deploy_k8s.yaml

@@ -0,0 +1,25 @@
+---
+- name: Deploy k8s cluster
+  hosts: localhost
+  become: true
+  become_method: sudo
+  gather_facts: false
+
+  pre_tasks:
+    - name: install python 2
+      raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal)
+
+    - name: Gather facts after python installation
+      setup: filter=ansible_*
+
+    - name: Install packages
+      apt: name={{ item }} state=present update_cache=yes
+      with_items:
+        - ansible
+        - unzip
+        - httpie
+
+  roles:
+    - docker
+    - k8s_cli
+    - kube_master

tools/gate/kubeadm/playbook/roles/docker/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+docker_version: 17.03.2~ce-0~ubuntu-xenial

tools/gate/kubeadm/playbook/roles/docker/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+- name: Install packages
+  apt: name={{ item }} state=present update_cache=yes
+  with_items:
+    - apt-transport-https
+    - ca-certificates
+    - curl
+
+- name: Add Docker’s official GPG key
+  apt_key:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    state: present
+
+- name: Set up the stable repository
+  apt_repository: repo='deb https://download.docker.com/linux/ubuntu xenial stable' state=present
+
+- name: Install docker-ce
+  apt:
+    name: docker-ce={{ docker_version }}
+    state: present
+    update_cache: yes

tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/install_kube_prompt_tasks.yml

@@ -0,0 +1,26 @@
+---
+- name: Create temporary directory
+  file:
+    state: directory
+    path: ~/kube-prompt
+  register: dir
+
+- name: Download kube-prompt
+  get_url:
+    url: https://github.com/c-bata/kube-prompt/releases/download/v1.0.3/kube-prompt_v1.0.3_linux_amd64.zip
+    dest: "{{ dir.path }}/kube-prompt_v1.0.3_linux_amd64.zip"
+    force: no
+
+- name: Unarchive kube-prompt
+  unarchive:
+    src: "{{ dir.path }}/kube-prompt_v1.0.3_linux_amd64.zip"
+    dest: "{{ dir.path }}"
+    remote_src: yes
+    creates: "{{ dir.path }}/kube-prompt"
+
+- name: Copy kube-prompt to bin
+  copy:
+    dest: /usr/local/bin/kprompt
+    src: "{{ dir.path }}/kube-prompt"
+    mode: u+x
+    remote_src: yes

tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/main.yml

@@ -0,0 +1,20 @@
+---
+- name: Add Kubernetes GPG key
+  apt_key:
+    url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
+    state: present
+
+- name: Set up Kubernetes repository
+  apt_repository: repo='deb http://apt.kubernetes.io/ kubernetes-xenial main' state=present
+
+- name: Install kube CLIs
+  apt:
+    name: "{{ item }}"
+    state: present
+    update_cache: yes
+  with_items:
+    - kubelet
+    - kubeadm
+    - kubectl
+
+- import_tasks: install_kube_prompt_tasks.yml

tools/gate/kubeadm/playbook/roles/kube_master/tasks/main.yml

@@ -0,0 +1,46 @@
+---
+- name: disable swap
+  shell: swapoff -a
+
+- name: drain the node if needed
+  shell: kubectl drain {{ ansible_nodename }} --delete-local-data --force --ignore-daemonsets
+  ignore_errors: True
+
+- name: delete node if needed
+  shell: kubectl delete node {{ ansible_nodename }}
+  ignore_errors: True
+
+- name: reset kubeadm setup
+  shell: kubeadm reset
+  ignore_errors: True
+
+- name: Init master
+  shell: kubeadm init --pod-network-cidr=192.168.0.0/16 --kubernetes-version=1.9.3
+  register: kubeadm_init
+
+- fail:
+    msg: "kubeadm init command failed."
+  when: kubeadm_init.stdout.find ("kubeadm join") == -1
+
+- name: Store kubeadm join string
+  set_fact:
+    join_str: "{{ kubeadm_init.stdout | regex_search('kubeadm join(.*)$') }}"
+
+- name: Prepare kube config
+  shell: mkdir -p {{ item }}/.kube && cp -a /etc/kubernetes/admin.conf {{ item }}/.kube/config && chmod 755 {{ item }}/.kube/config
+  with_items:
+    - "{{ ansible_env.HOME }}"
+    - /home/ubuntu
+    - /opt/stack/new
+
+- name: Allow pod on master
+  shell: kubectl taint nodes --all node-role.kubernetes.io/master-
+
+- name: Download calico yaml
+  get_url:
+    url: https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml
+    dest: "{{ ansible_env.HOME }}/calico.yaml"
+    force: no
+
+- name: Install network plugin
+  shell: kubectl apply -f {{ ansible_env.HOME }}/calico.yaml

tools/gate/kubeadm/setup_gate.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+export WORK_DIR=$(pwd)
+source ${WORK_DIR}/tools/gate/kubeadm/vars.sh
+source ${WORK_DIR}/tools/gate/kubeadm/funcs/common.sh
+source ${WORK_DIR}/tools/gate/kubeadm/funcs/network.sh
+
+# Do the basic node setup for running the gate
+gate_base_setup
+net_resolv_pre_kube
+net_hosts_pre_kube
+
+# Setup the K8s Cluster
+sudo apt-add-repository --yes ppa:ansible/ansible && sudo apt-get update -y -qq && sudo apt-get install ansible -y -qq
+ansible-playbook ${WORK_DIR}/tools/gate/kubeadm/playbook/deploy_k8s.yaml
+
+# waits until kubectl can access the api server
+sudo chown $(id -u):$(id -g) ${HOME}/.kube/config
+end=$(($(date +%s) + 600))
+READY="False"
+while true; do
+  READY=$(kubectl get nodes --no-headers=true | awk "{ print \$2 }" | head -1)
+  [ "$READY" == "Ready" ] && break || true
+  sleep 2
+  now=$(date +%s)
+  [ $now -gt $end ] && echo "Failed to setup kubernetes cluster in time" && exit -1
+done
+
+create_k8s_screen
+
+#net_hosts_post_kube
+#net_resolv_post_kube

tools/gate/kubeadm/vars.sh

@@ -0,0 +1,19 @@
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set work dir if not already done
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
+
+# Get Host OS
+source /etc/os-release
+export HOST_OS=${HOST_OS:="${ID}"}
+
+# Set versions of K8s to use
+export KUBE_VERSION=${KUBE_VERSION:-"v1.8.0"}
+
+# Set Upstream DNS
+export UPSTREAM_DNS=${UPSTREAM_DNS:-"8.8.8.8"}