KubeADM-AIO: Update Image to support Multinode Operation Natively

This PS also pins the remaining components of the kubelet, moves the K8s version to 1.6.4 and restores CentOS 7 operation. Change-Id: Ia32c9e02dbf9451c13addce436f6e36f5a0a622f
2017-06-04 11:49:17 -05:00 · 2017-06-04 11:49:17 -05:00 · 1ca8efa3a6
commit 1ca8efa3a6
parent c1e008643d
9 changed files with 84 additions and 54 deletions
--- a/tools/gate/funcs/kube.sh
+++ b/tools/gate/funcs/kube.sh
@ -56,11 +56,16 @@ function kubeadm_aio_reqs_install {
    sudo yum install -y \
            epel-release
    sudo yum install -y \
-            docker \
+            docker-latest \
            nfs-utils \
            jq
-    sudo cp -f /usr/lib/systemd/system/docker.service /etc/systemd/system/docker.service
+    sudo cp -f /usr/lib/systemd/system/docker-latest.service /etc/systemd/system/docker.service
+    sudo sed -i "s|/var/lib/docker-latest|/var/lib/docker|g" /etc/systemd/system/docker.service
+    sudo sed -i 's/^OPTIONS/#OPTIONS/g' /etc/sysconfig/docker-latest
    sudo sed -i "s|^MountFlags=slave|MountFlags=share|g" /etc/systemd/system/docker.service
+    sudo sed -i "/--seccomp-profile/,+1 d" /etc/systemd/system/docker.service
+    echo "DOCKER_STORAGE_OPTIONS=--storage-driver=overlay" | sudo tee /etc/sysconfig/docker-latest-storage
+    sudo setenforce 0 || true
    sudo systemctl daemon-reload
    sudo systemctl restart docker
  elif [ "x$HOST_OS" == "xfedora" ]; then
@ -71,6 +76,7 @@ function kubeadm_aio_reqs_install {
    sudo cp -f /usr/lib/systemd/system/docker-latest.service /etc/systemd/system/docker.service
    sudo sed -i "s|/var/lib/docker-latest|/var/lib/docker|g" /etc/systemd/system/docker.service
    echo "DOCKER_STORAGE_OPTIONS=--storage-driver=overlay2" | sudo tee /etc/sysconfig/docker-latest-storage
+    sudo setenforce 0 || true
    sudo systemctl daemon-reload
    sudo systemctl restart docker
  fi
@ -87,11 +93,6 @@ function kubeadm_aio_build {
 }

 function kubeadm_aio_launch {
-  if [ "x$HOST_OS" == "xcentos" ]; then
-    sudo setenforce 0 || true
-  elif [ "x$HOST_OS" == "xfedora" ]; then
-    sudo setenforce 0 || true
-  fi
  ${WORK_DIR}/tools/kubeadm-aio/kubeadm-aio-launcher.sh
  mkdir -p ${HOME}/.kube
  cat ${KUBECONFIG} > ${HOME}/.kube/config
--- a/tools/gate/kubeadm_aio.sh
+++ b/tools/gate/kubeadm_aio.sh
@ -17,6 +17,6 @@ source ${WORK_DIR}/tools/gate/funcs/network.sh
 source ${WORK_DIR}/tools/gate/funcs/kube.sh

 kubeadm_aio_reqs_install
-#kubeadm_aio_build
+sudo docker pull ${KUBEADM_IMAGE} || kubeadm_aio_build
 kubeadm_aio_launch
 net_resolv_kube
--- a/tools/gate/setup_gate.sh
+++ b/tools/gate/setup_gate.sh
@ -14,9 +14,9 @@
 set -ex

 export HELM_VERSION=${2:-v2.3.0}
-export KUBE_VERSION=${3:-v1.6.2}
+export KUBE_VERSION=${3:-v1.6.4}
 export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf
-export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6
+export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}

 export WORK_DIR=$(pwd)
 source /etc/os-release
--- a/tools/kubeadm-aio/Dockerfile
+++ b/tools/kubeadm-aio/Dockerfile
@ -1,14 +1,15 @@
 FROM ubuntu:16.04
 MAINTAINER pete.birley@att.com

-ARG HELM_VERSION=v2.3.0
-ARG KUBE_VERSION=v1.6.2
-
-ENV container="docker" \
+ENV HELM_VERSION=v2.3.0 \
+    KUBE_VERSION=v1.6.4 \
+    CNI_VERSION=v0.5.2 \
+    container="docker" \
    DEBIAN_FRONTEND="noninteractive"

 RUN set -x \
-    && cd /tmp \
+    && TMP_DIR=$(mktemp --directory) \
+    && cd ${TMP_DIR} \
    && apt-get update \
    && apt-get install -y \
        apt-transport-https \
@ -31,19 +32,29 @@ RUN set -x \
    && dpkg --configure kubeadm \
    && apt-get install -yf kubeadm \
    && mkdir -p /etc/kubernetes/manifests \
+# Install kubectl:
+    && curl -sSL https://dl.k8s.io/${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
+    && mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \
+    && chmod +x /usr/bin/kubectl \
 # Install kubelet & kubeadm binaries:
 # (portdirect) We do things in this weird way to let us use the deps and systemd
 # units from the packages in the .deb repo.
    && curl -sSL https://dl.k8s.io/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
-    && mv /tmp/server/bin/kubelet /usr/bin/kubelet \
+    && mv ${TMP_DIR}/server/bin/kubelet /usr/bin/kubelet \
    && chmod +x /usr/bin/kubelet \
-    && mv /tmp/server/bin/kubeadm /usr/bin/kubeadm \
+    && mv ${TMP_DIR}/server/bin/kubeadm /usr/bin/kubeadm \
    && chmod +x /usr/bin/kubeadm \
+# Install CNI:
+    && CNI_BIN_DIR=/opt/cni/bin \
+    && mkdir -p ${CNI_BIN_DIR} \
+    && cd ${CNI_BIN_DIR} \
+    && curl -sSL https://github.com/containernetworking/cni/releases/download/$CNI_VERSION/cni-amd64-$CNI_VERSION.tgz | tar -zxv --strip-components=1 \
+    && cd ${TMP_DIR} \
 # Move kubelet binary as we will run containerised
    && mv /usr/bin/kubelet /usr/bin/kubelet-real \
 # Install helm binary
    && curl -sSL https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
-    && mv /tmp/helm /usr/bin/helm \
+    && mv ${TMP_DIR}/helm /usr/bin/helm \
 # Install openstack-helm dev utils
    && apt-get install -y \
        make \
@ -64,7 +75,7 @@ RUN set -x \
    && rm -rf /var/lib/apt/lists/* \
 # Clean up tmp dir
    && cd / \
-    && rm -rf /tmp/*
+    && rm -rf ${TMP_DIR}

 # Load assets into place, setup startup target & units
 COPY ./assets/ /
--- a/tools/kubeadm-aio/README.rst
+++ b/tools/kubeadm-aio/README.rst
@ -31,7 +31,7 @@ You should install the ``kubectl`` and ``helm`` binaries:

 .. code:: bash

-    KUBE_VERSION=v1.6.0
+    KUBE_VERSION=v1.6.4
    HELM_VERSION=v2.3.0

    TMP_DIR=$(mktemp -d)
@ -57,7 +57,7 @@ repo run:

 .. code:: bash

-    export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6
+    export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.4
    sudo docker build --pull -t ${KUBEADM_IMAGE} tools/kubeadm-aio

 Deploy the AIO environment
@ -67,8 +67,8 @@ To launch the environment then run:

 .. code:: bash

-    export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6
-    export KUBE_VERSION=v1.6.2
+    export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.4
+    export KUBE_VERSION=v1.6.4
    ./tools/kubeadm-aio/kubeadm-aio-launcher.sh
    export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf

--- a/tools/kubeadm-aio/assets/etc/kube-role
+++ b/tools/kubeadm-aio/assets/etc/kube-role
@ -0,0 +1,3 @@
+# If KUBE_ROLE is set 'master' kubeadm-aio will set this node up to be a master
+# node, otherwise if 'worker', will join an existing cluster.
+KUBE_ROLE=master
--- a/tools/kubeadm-aio/assets/etc/kubeadm-join-command-args
+++ b/tools/kubeadm-aio/assets/etc/kubeadm-join-command-args
@ -0,0 +1 @@
+KUBEADM_JOIN_ARGS="no_command_supplied"
--- a/tools/kubeadm-aio/assets/kubeadm-aio
+++ b/tools/kubeadm-aio/assets/kubeadm-aio
@ -25,7 +25,7 @@ echo 'Setting up K8s version to deploy'
 : ${KUBE_VERSION:="default"}
 sed -i "s|KUBE_VERSION=.*|KUBE_VERSION=${KUBE_VERSION}|g" /etc/kube-version

-echo 'Setting up device image to use for kube-api'
+echo 'Setting up device to use for kube-api'
 : ${KUBE_BIND_DEV:="autodetect"}
 sed -i "s|KUBE_BIND_DEV=.*|KUBE_BIND_DEV=${KUBE_BIND_DEV}|g" /etc/kubeapi-device

@ -33,5 +33,13 @@ echo 'Setting up container image to use for kubelet'
 : ${KUBELET_CONTAINER:="this_one"}
 sed -i "s|KUBELET_CONTAINER=.*|KUBELET_CONTAINER=${KUBELET_CONTAINER}|g" /etc/kubelet-container

+echo 'Setting whether this node is a master, or slave, K8s node'
+: ${KUBE_ROLE:="master"}
+sed -i "s|KUBE_ROLE=.*|KUBE_ROLE=${KUBE_ROLE}|g" /etc/kube-role
+
+echo 'Setting any kubeadm join commands'
+: ${KUBEADM_JOIN_ARGS:="no_command_supplied"}
+sed -i "s|KUBEADM_JOIN_ARGS=.*|KUBEADM_JOIN_ARGS=\"${KUBEADM_JOIN_ARGS}\"|g" /etc/kubeadm-join-command-args
+
 echo 'Starting Systemd'
 exec /bin/systemd --system
--- a/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio
+++ b/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio
@ -15,34 +15,40 @@
 #    under the License.
 set -xe

-# Define k8s version
-source /etc/kube-version
-if [[ "${KUBE_VERSION}" == "default" ]]; then
+source /etc/kube-role
+if [[ "${KUBE_ROLE}" == "master" ]]; then
+  # Define k8s version
+  source /etc/kube-version
+  if [[ "${KUBE_VERSION}" == "default" ]]; then
    KUBE_VERSION_FLAG=""
-else
+  else
    KUBE_VERSION_FLAG="--kubernetes-version=${KUBE_VERSION}"
    echo "We will use K8s ${KUBE_VERSION}"
-fi
+  fi

-echo 'Setting up K8s'
-source /etc/kubeapi-device
-if [[ "$KUBE_BIND_DEV" != "autodetect" ]]; then
+  echo 'Setting up K8s'
+  source /etc/kubeapi-device
+  if [[ "$KUBE_BIND_DEV" != "autodetect" ]]; then
    KUBE_BIND_IP=$(ip addr list ${KUBE_BIND_DEV} |grep "inet " |cut -d' ' -f6|cut -d/ -f1)
    echo 'We are going to bind the K8s API to: ${KUBE_BIND_IP}'
    kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --api-advertise-addresses ${KUBE_BIND_IP}
-else
+  else
    kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG}
+  fi
+
+  echo 'Setting up K8s client'
+  cp /etc/kubernetes/admin.conf /root/
+  export KUBECONFIG=/root/admin.conf
+
+  echo 'Marking master node as schedulable'
+  kubectl taint nodes --all node-role.kubernetes.io/master-
+
+  echo 'Installing Calico CNI'
+  kubectl apply -f /opt/cni-manifests/calico.yaml
+
+  echo 'Setting Up Cluser for OpenStack-Helm dev use'
+  /usr/bin/openstack-helm-dev-prep
+elif [[ "${KUBE_ROLE}" == "worker" ]]; then
+  source /etc/kubeadm-join-command-args
+  kubeadm join --skip-preflight-checks ${KUBEADM_JOIN_ARGS}
 fi
-
-echo 'Setting up K8s client'
-cp /etc/kubernetes/admin.conf /root/
-export KUBECONFIG=/root/admin.conf
-
-echo 'Marking master node as schedulable'
-kubectl taint nodes --all node-role.kubernetes.io/master-
-
-echo 'Installing Calico CNI'
-kubectl apply -f /opt/cni-manifests/calico.yaml
-
-echo 'Setting Up Cluser for OpenStack-Helm dev use'
-/usr/bin/openstack-helm-dev-prep