KubeADM-AIO: Make init more stable and support multiple PVC backends

This PS makes the init of the KubeADM environment more stable by ensuring
all kube-system pods are up before trying to launch tiller. the NFS PVC
provider is now made optional but enabled by default, so that Ceph (and
other more complex providers) can be supported in future. Finaly the
scripts to bring up k8s/helm are refined to not bring in the clients if
already present with the correct versions on the host.

Change-Id: I4d7ceb6196e8fd5e3350ec99f09fbe6bb5fe01f6

tools/gate/funcs/helm.sh

@@ -14,7 +14,6 @@
 set -e
 
 function helm_install {
-  TMP_DIR=$(mktemp -d)
   if [ "x$HOST_OS" == "xubuntu" ]; then
     sudo apt-get update -y
     sudo apt-get install -y --no-install-recommends -qq \
@@ -35,10 +34,14 @@ function helm_install {
   fi
 
   # install helm
-  curl -sSL https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR}
-  sudo mv ${TMP_DIR}/helm /usr/local/bin/helm
-
-  rm -rf ${TMP_DIR}
+  if CURRENT_HELM_LOC=$(type -p helm); then
+    CURRENT_HELM_VERSION=$(${CURRENT_HELM_LOC} version --client --short | awk '{ print $NF }' | awk -F '+' '{ print $1 }')
+  fi
+  [ "x$HELM_VERSION" == "x$CURRENT_HELM_VERSION" ] || ( \
+    TMP_DIR=$(mktemp -d)
+    curl -sSL https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR}
+    sudo mv ${TMP_DIR}/helm /usr/local/bin/helm
+    rm -rf ${TMP_DIR} )
 }
 
 function helm_serve {

tools/gate/funcs/kube.sh

@@ -72,7 +72,6 @@ function kube_wait_for_nodes {
 }
 
 function kubeadm_aio_reqs_install {
-  TMP_DIR=$(mktemp -d)
   if [ "x$HOST_OS" == "xubuntu" ]; then
     sudo apt-get update -y
     sudo apt-get install -y --no-install-recommends -qq \
@@ -108,11 +107,16 @@ function kubeadm_aio_reqs_install {
     sudo systemctl restart docker
   fi
 
-  curl -sSL https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl
-  chmod +x ${TMP_DIR}/kubectl
-  sudo mv ${TMP_DIR}/kubectl /usr/local/bin/kubectl
+  if CURRENT_KUBECTL_LOC=$(type -p kubectl); then
+    CURRENT_KUBECTL_VERSION=$(${CURRENT_KUBECTL_LOC} version --client --short | awk '{ print $NF }' | awk -F '+' '{ print $1 }')
+  fi
+  [ "x$KUBE_VERSION" == "x$CURRENT_KUBECTL_VERSION" ] || ( \
+    TMP_DIR=$(mktemp -d)
+    curl -sSL https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl
+    chmod +x ${TMP_DIR}/kubectl
+    sudo mv ${TMP_DIR}/kubectl /usr/local/bin/kubectl
+    rm -rf ${TMP_DIR} )
 
-  rm -rf ${TMP_DIR}
 }
 
 function kubeadm_aio_build {

tools/kubeadm-aio/Dockerfile

@@ -60,6 +60,7 @@ RUN set -x \
         make \
         git \
         vim \
+        jq \
 # Install nfs utils for development PVC provisioner
         nfs-common \
 # Tweak Systemd units and targets for running in a container

tools/kubeadm-aio/assets/usr/bin/wait-for-kube-nodes

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+set -e
+
+# Default wait timeout is 180 seconds
+: ${KUBECONFIG:="/etc/kubernetes/admin.conf"}
+export KUBECONFIG=${KUBECONFIG}
+
+end=$(date +%s)
+if [ x$2 != "x" ]; then
+ end=$((end + $2))
+else
+ end=$((end + 180))
+fi
+while true; do
+    NUMBER_OF_NODES=$(kubectl get nodes --no-headers -o name | wc -l)
+    NUMBER_OF_NODES_EXPECTED=$(($(cat /etc/nodepool/sub_nodes_private | wc -l) + 1))
+    [ $NUMBER_OF_NODES -eq $NUMBER_OF_NODES_EXPECTED ] && \
+        NODES_ONLINE="True" || NODES_ONLINE="False"
+    while read SUB_NODE; do
+      echo $SUB_NODE | grep -q ^Ready && NODES_READY="True" || NODES_READY="False"
+    done < <(kubectl get nodes --no-headers | awk '{ print $2 }')
+    [ $NODES_ONLINE == "True" -a $NODES_READY == "True"  ] && \
+        break || true
+    sleep 5
+    now=$(date +%s)
+    [ $now -gt $end ] && echo "Nodes Failed to be ready in time." && \
+        kubectl get nodes -o wide && exit -1
+done

tools/kubeadm-aio/assets/usr/bin/wait-for-kube-pods

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+set -e
+
+# From Kolla-Kubernetes, orginal authors Kevin Fox & Serguei Bezverkhi
+# Default wait timeout is 180 seconds
+: ${KUBECONFIG:="/etc/kubernetes/admin.conf"}
+export KUBECONFIG=${KUBECONFIG}
+
+end=$(date +%s)
+if [ x$2 != "x" ]; then
+ end=$((end + $2))
+else
+ end=$((end + 180))
+fi
+while true; do
+    kubectl get pods --namespace=$1 -o json | jq -r \
+        '.items[].status.phase' | grep Pending > /dev/null && \
+        PENDING=True || PENDING=False
+    query='.items[]|select(.status.phase=="Running")'
+    query="$query|.status.containerStatuses[].ready"
+    kubectl get pods --namespace=$1 -o json | jq -r "$query" | \
+        grep false > /dev/null && READY="False" || READY="True"
+    kubectl get jobs -o json --namespace=$1 | jq -r \
+        '.items[] | .spec.completions == .status.succeeded' | \
+        grep false > /dev/null && JOBR="False" || JOBR="True"
+    [ $PENDING == "False" -a $READY == "True" -a $JOBR == "True" ] && \
+        break || true
+    sleep 1
+    now=$(date +%s)
+    [ $now -gt $end ] && echo containers failed to start. && \
+        kubectl get pods --namespace $1 -o wide && exit -1
+done

tools/kubeadm-aio/kubeadm-aio-launcher.sh

@@ -89,11 +89,17 @@ while true; do
 done
 set -x
 
+# Waiting for kube-system pods to be ready before continuing
+sudo docker exec kubeadm-aio wait-for-kube-pods kube-system
+
 # Initialize Helm
 helm init
 
 # Initialize Environment for Development
 sudo docker exec kubeadm-aio openstack-helm-dev-prep
 
-# Deploy NFS provisioner into enviromment
-sudo docker exec kubeadm-aio openstack-helm-nfs-prep
+: ${PVC_BACKEND:="nfs"}
+if [ "$PVC_BACKEND" == "nfs" ]; then
+  # Deploy NFS provisioner into enviromment
+  sudo docker exec kubeadm-aio openstack-helm-nfs-prep
+fi