KubeADM-AIO: Update Image to support Multinode Operation Natively

This PS also pins the remaining components of the kubelet, moves
the K8s version to 1.6.4 and restores CentOS 7 operation.

Change-Id: Ia32c9e02dbf9451c13addce436f6e36f5a0a622f

tools/gate/funcs/kube.sh

@@ -56,11 +56,16 @@ function kubeadm_aio_reqs_install {
     sudo yum install -y \
             epel-release
     sudo yum install -y \
-            docker \
+            docker-latest \
             nfs-utils \
             jq
-    sudo cp -f /usr/lib/systemd/system/docker.service /etc/systemd/system/docker.service
+    sudo cp -f /usr/lib/systemd/system/docker-latest.service /etc/systemd/system/docker.service
+    sudo sed -i "s|/var/lib/docker-latest|/var/lib/docker|g" /etc/systemd/system/docker.service
+    sudo sed -i 's/^OPTIONS/#OPTIONS/g' /etc/sysconfig/docker-latest
     sudo sed -i "s|^MountFlags=slave|MountFlags=share|g" /etc/systemd/system/docker.service
+    sudo sed -i "/--seccomp-profile/,+1 d" /etc/systemd/system/docker.service
+    echo "DOCKER_STORAGE_OPTIONS=--storage-driver=overlay" | sudo tee /etc/sysconfig/docker-latest-storage
+    sudo setenforce 0 || true
     sudo systemctl daemon-reload
     sudo systemctl restart docker
   elif [ "x$HOST_OS" == "xfedora" ]; then
@@ -71,6 +76,7 @@ function kubeadm_aio_reqs_install {
     sudo cp -f /usr/lib/systemd/system/docker-latest.service /etc/systemd/system/docker.service
     sudo sed -i "s|/var/lib/docker-latest|/var/lib/docker|g" /etc/systemd/system/docker.service
     echo "DOCKER_STORAGE_OPTIONS=--storage-driver=overlay2" | sudo tee /etc/sysconfig/docker-latest-storage
+    sudo setenforce 0 || true
     sudo systemctl daemon-reload
     sudo systemctl restart docker
   fi
@@ -87,11 +93,6 @@ function kubeadm_aio_build {
 }
 
 function kubeadm_aio_launch {
-  if [ "x$HOST_OS" == "xcentos" ]; then
-    sudo setenforce 0 || true
-  elif [ "x$HOST_OS" == "xfedora" ]; then
-    sudo setenforce 0 || true
-  fi
   ${WORK_DIR}/tools/kubeadm-aio/kubeadm-aio-launcher.sh
   mkdir -p ${HOME}/.kube
   cat ${KUBECONFIG} > ${HOME}/.kube/config

tools/gate/kubeadm_aio.sh

@@ -17,6 +17,6 @@ source ${WORK_DIR}/tools/gate/funcs/network.sh
 source ${WORK_DIR}/tools/gate/funcs/kube.sh
 
 kubeadm_aio_reqs_install
-#kubeadm_aio_build
+sudo docker pull ${KUBEADM_IMAGE} || kubeadm_aio_build
 kubeadm_aio_launch
 net_resolv_kube

tools/gate/setup_gate.sh

@@ -14,9 +14,9 @@
 set -ex
 
 export HELM_VERSION=${2:-v2.3.0}
-export KUBE_VERSION=${3:-v1.6.2}
+export KUBE_VERSION=${3:-v1.6.4}
 export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf
-export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6
+export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}
 
 export WORK_DIR=$(pwd)
 source /etc/os-release

tools/kubeadm-aio/Dockerfile

@@ -1,14 +1,15 @@
 FROM ubuntu:16.04
 MAINTAINER pete.birley@att.com
 
-ARG HELM_VERSION=v2.3.0
+ENV HELM_VERSION=v2.3.0 \
-ARG KUBE_VERSION=v1.6.2
+    KUBE_VERSION=v1.6.4 \
-
+    CNI_VERSION=v0.5.2 \
-ENV container="docker" \
+    container="docker" \
     DEBIAN_FRONTEND="noninteractive"
 
 RUN set -x \
-    && cd /tmp \
+    && TMP_DIR=$(mktemp --directory) \
+    && cd ${TMP_DIR} \
     && apt-get update \
     && apt-get install -y \
         apt-transport-https \
@@ -31,19 +32,29 @@ RUN set -x \
     && dpkg --configure kubeadm \
     && apt-get install -yf kubeadm \
     && mkdir -p /etc/kubernetes/manifests \
+# Install kubectl:
+    && curl -sSL https://dl.k8s.io/${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
+    && mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \
+    && chmod +x /usr/bin/kubectl \
 # Install kubelet & kubeadm binaries:
 # (portdirect) We do things in this weird way to let us use the deps and systemd
 # units from the packages in the .deb repo.
     && curl -sSL https://dl.k8s.io/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
-    && mv /tmp/server/bin/kubelet /usr/bin/kubelet \
+    && mv ${TMP_DIR}/server/bin/kubelet /usr/bin/kubelet \
     && chmod +x /usr/bin/kubelet \
-    && mv /tmp/server/bin/kubeadm /usr/bin/kubeadm \
+    && mv ${TMP_DIR}/server/bin/kubeadm /usr/bin/kubeadm \
     && chmod +x /usr/bin/kubeadm \
+# Install CNI:
+    && CNI_BIN_DIR=/opt/cni/bin \
+    && mkdir -p ${CNI_BIN_DIR} \
+    && cd ${CNI_BIN_DIR} \
+    && curl -sSL https://github.com/containernetworking/cni/releases/download/$CNI_VERSION/cni-amd64-$CNI_VERSION.tgz | tar -zxv --strip-components=1 \
+    && cd ${TMP_DIR} \
 # Move kubelet binary as we will run containerised
     && mv /usr/bin/kubelet /usr/bin/kubelet-real \
 # Install helm binary
     && curl -sSL https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
-    && mv /tmp/helm /usr/bin/helm \
+    && mv ${TMP_DIR}/helm /usr/bin/helm \
 # Install openstack-helm dev utils
     && apt-get install -y \
         make \
@@ -64,7 +75,7 @@ RUN set -x \
     && rm -rf /var/lib/apt/lists/* \
 # Clean up tmp dir
     && cd / \
-    && rm -rf /tmp/*
+    && rm -rf ${TMP_DIR}
 
 # Load assets into place, setup startup target & units
 COPY ./assets/ /

tools/kubeadm-aio/README.rst

@@ -31,7 +31,7 @@ You should install the ``kubectl`` and ``helm`` binaries:
 
 .. code:: bash
 
-    KUBE_VERSION=v1.6.0
+    KUBE_VERSION=v1.6.4
     HELM_VERSION=v2.3.0
 
     TMP_DIR=$(mktemp -d)
@@ -57,7 +57,7 @@ repo run:
 
 .. code:: bash
 
-    export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6
+    export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.4
     sudo docker build --pull -t ${KUBEADM_IMAGE} tools/kubeadm-aio
 
 Deploy the AIO environment
@@ -67,8 +67,8 @@ To launch the environment then run:
 
 .. code:: bash
 
-    export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6
+    export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.4
-    export KUBE_VERSION=v1.6.2
+    export KUBE_VERSION=v1.6.4
     ./tools/kubeadm-aio/kubeadm-aio-launcher.sh
     export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf
 

tools/kubeadm-aio/assets/etc/kube-role

@@ -0,0 +1,3 @@
+# If KUBE_ROLE is set 'master' kubeadm-aio will set this node up to be a master
+# node, otherwise if 'worker', will join an existing cluster.
+KUBE_ROLE=master

tools/kubeadm-aio/assets/etc/kubeadm-join-command-args

@@ -0,0 +1 @@
+KUBEADM_JOIN_ARGS="no_command_supplied"

tools/kubeadm-aio/assets/kubeadm-aio

@@ -25,7 +25,7 @@ echo 'Setting up K8s version to deploy'
 : ${KUBE_VERSION:="default"}
 sed -i "s|KUBE_VERSION=.*|KUBE_VERSION=${KUBE_VERSION}|g" /etc/kube-version
 
-echo 'Setting up device image to use for kube-api'
+echo 'Setting up device to use for kube-api'
 : ${KUBE_BIND_DEV:="autodetect"}
 sed -i "s|KUBE_BIND_DEV=.*|KUBE_BIND_DEV=${KUBE_BIND_DEV}|g" /etc/kubeapi-device
 
@@ -33,5 +33,13 @@ echo 'Setting up container image to use for kubelet'
 : ${KUBELET_CONTAINER:="this_one"}
 sed -i "s|KUBELET_CONTAINER=.*|KUBELET_CONTAINER=${KUBELET_CONTAINER}|g" /etc/kubelet-container
 
+echo 'Setting whether this node is a master, or slave, K8s node'
+: ${KUBE_ROLE:="master"}
+sed -i "s|KUBE_ROLE=.*|KUBE_ROLE=${KUBE_ROLE}|g" /etc/kube-role
+
+echo 'Setting any kubeadm join commands'
+: ${KUBEADM_JOIN_ARGS:="no_command_supplied"}
+sed -i "s|KUBEADM_JOIN_ARGS=.*|KUBEADM_JOIN_ARGS=\"${KUBEADM_JOIN_ARGS}\"|g" /etc/kubeadm-join-command-args
+
 echo 'Starting Systemd'
 exec /bin/systemd --system

tools/kubeadm-aio/assets/usr/bin/kubeadm-aio

@@ -15,34 +15,40 @@
 #    under the License.
 set -xe
 
-# Define k8s version
+source /etc/kube-role
-source /etc/kube-version
+if [[ "${KUBE_ROLE}" == "master" ]]; then
-if [[ "${KUBE_VERSION}" == "default" ]]; then
+  # Define k8s version
+  source /etc/kube-version
+  if [[ "${KUBE_VERSION}" == "default" ]]; then
     KUBE_VERSION_FLAG=""
-else
+  else
     KUBE_VERSION_FLAG="--kubernetes-version=${KUBE_VERSION}"
     echo "We will use K8s ${KUBE_VERSION}"
-fi
+  fi
 
-echo 'Setting up K8s'
+  echo 'Setting up K8s'
-source /etc/kubeapi-device
+  source /etc/kubeapi-device
-if [[ "$KUBE_BIND_DEV" != "autodetect" ]]; then
+  if [[ "$KUBE_BIND_DEV" != "autodetect" ]]; then
     KUBE_BIND_IP=$(ip addr list ${KUBE_BIND_DEV} |grep "inet " |cut -d' ' -f6|cut -d/ -f1)
     echo 'We are going to bind the K8s API to: ${KUBE_BIND_IP}'
     kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --api-advertise-addresses ${KUBE_BIND_IP}
-else
+  else
     kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG}
+  fi
+
+  echo 'Setting up K8s client'
+  cp /etc/kubernetes/admin.conf /root/
+  export KUBECONFIG=/root/admin.conf
+
+  echo 'Marking master node as schedulable'
+  kubectl taint nodes --all node-role.kubernetes.io/master-
+
+  echo 'Installing Calico CNI'
+  kubectl apply -f /opt/cni-manifests/calico.yaml
+
+  echo 'Setting Up Cluser for OpenStack-Helm dev use'
+  /usr/bin/openstack-helm-dev-prep
+elif [[ "${KUBE_ROLE}" == "worker" ]]; then
+  source /etc/kubeadm-join-command-args
+  kubeadm join --skip-preflight-checks ${KUBEADM_JOIN_ARGS}
 fi
-
-echo 'Setting up K8s client'
-cp /etc/kubernetes/admin.conf /root/
-export KUBECONFIG=/root/admin.conf
-
-echo 'Marking master node as schedulable'
-kubectl taint nodes --all node-role.kubernetes.io/master-
-
-echo 'Installing Calico CNI'
-kubectl apply -f /opt/cni-manifests/calico.yaml
-
-echo 'Setting Up Cluser for OpenStack-Helm dev use'
-/usr/bin/openstack-helm-dev-prep