Optimizing multinode job at the gate

Optimizing multinode job at the gate

Change-Id: I59c342366113b4826bdd8b0916152e4ed32c0a55
Partially-Implements: blueprint gate-continous-improvement

tests/bin/setup_canal.sh

@@ -13,6 +13,4 @@ sed -i "s@10.96.232.136@172.16.128.100@" /tmp/canal.yaml
 kubectl create -f /tmp/canal.yaml
 
 $DIR/tools/pull_containers.sh kube-system
-$DIR/tools/wait_for_pods.sh kube-system
-
-$DIR/tools/test_kube_dns.sh
+$DIR/tools/wait_for_pods.sh kube-system 240

tools/setup_gate_ceph.sh

@@ -36,11 +36,14 @@ tests/bin/setup_gate_loopback.sh
 
 tools/setup_kubernetes.sh master
 
-kubectl describe nodes
-
 kubectl taint nodes --all=true  node-role.kubernetes.io/master:NoSchedule-
 
-# Turn up kube-proxy logging
+#
+# Setting up networking on master, before slave nodes in multinode
+# scenario will attempt to join the cluster
+tests/bin/setup_canal.sh
+
+# Turn up kube-proxy logging enable only for debug 
 # kubectl -n kube-system get ds -l 'component=kube-proxy-amd64' -o json \
 #   | sed 's/--v=4/--v=9/' \
 #   | kubectl apply -f - && kubectl -n kube-system delete pods -l 'component=kube-proxy-amd64'
@@ -87,7 +90,10 @@ if [ "x$CONFIG" != "xceph-multi" ]; then
     kubectl label node $NODE kolla_compute=true
 fi
 
-tests/bin/setup_canal.sh
+tools/pull_containers.sh kube-system
+tools/wait_for_pods.sh kube-system
+
+tools/test_kube_dns.sh
 
 # Setting up Helm
 setup_helm_common

tools/setup_gate_iscsi.sh

@@ -28,15 +28,18 @@ tests/bin/setup_gate_loopback_lvm.sh
 
 tools/setup_kubernetes.sh master
 
-kubectl describe nodes
-
 kubectl taint nodes --all=true  node-role.kubernetes.io/master:NoSchedule-
 
+tests/bin/setup_canal.sh
+
 NODE=$(hostname -s)
 kubectl label node $NODE kolla_controller=true kolla_compute=true \
                          kolla_storage=true
 
-tests/bin/setup_canal.sh
+tools/pull_containers.sh kube-system
+tools/wait_for_pods.sh kube-system
+
+tools/test_kube_dns.sh
 
 # Setting up Helm
 setup_helm_common

tools/setup_kubernetes.sh

@@ -63,14 +63,7 @@ if [ "$1" == "master" ]; then
     mkdir -p ~/.kube
     sudo cp /etc/kubernetes/admin.conf ~/.kube/config
     sudo chown $(id -u):$(id -g) ~/.kube/config
-    set +e
-    count=0
-    while true; do
-        kubectl get pods -n kube-system > /dev/null 2>&1 && break || true
-        sleep 1
-        count=$((count + 1))
-        [ $count -gt 30 ] && echo kube-apiserver failed to come back up. && exit -1
-    done
+    tools/wait_for_kube_control_plane.sh
 
 # NOTE(sbezverk/kfox111) This is a horible hack to get k8s 1.6 working. This should be
 # removed in favor of more fine grained rules.

tools/wait_for_kube_control_plane.sh

@@ -0,0 +1,35 @@
+#!/bin/bash -e
+# Default wait timeout is 180 seconds
+set +x
+count=0
+
+for x in etcd kube-apiserver kube-controller-manager kube-scheduler; do
+   echo $x
+   pod_name=""
+   while [ "x$pod_name" == "x" ]; do
+      pod_name=$(kubectl get pod -n kube-system | grep -G ^$x | awk '{print $1}')
+      if [ "x$pod_name" != "x" ]; then
+         while true; do
+            pod_status=$(kubectl get pod -n kube-system $pod_name -o jsonpath='{.status.phase}')
+            echo "Pod name: $pod_name status: $pod_status"
+            if [ "x$pod_status" != "xRunning" ]; then
+               if [ $count -gt 180 ]; then
+                  echo "Kubernetes cluster control plane failed to come up"
+                  exit -1
+               fi
+               count=$((count+1))
+               sleep 1
+            else
+               break
+            fi
+         done
+      else
+         if [ $count -gt 180 ]; then
+            echo "Kubernetes cluster control plane failed to come up"
+            exit -1
+         fi
+         count=$((count+1))
+         sleep 1
+      fi
+   done
+done