Merge "Script to smoke check kube"

tools/multi_nodes_gate/airship_gate/lib/ssh.sh

@@ -2,6 +2,10 @@ rsync_cmd() {
     rsync -e "ssh -F ${SSH_CONFIG_DIR}/config" "${@}"
 }
 
+ssh_cmd_raw() {
+    ssh -F "${SSH_CONFIG_DIR}/config" $@
+}
+
 ssh_cmd() {
     HOST=${1}
     shift

tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json

@@ -53,6 +53,11 @@
     {
       "name": "Deploy Site",
       "script": "shipyard-deploy-site.sh"
+    },
+    {
+      "name": "Validate Kube",
+      "script": "validate-kube.sh",
+      "on_error": "collect_genesis_info.sh"
     }
   ],
   "vm": {

tools/multi_nodes_gate/airship_gate/stages/validate-kube.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+if [[ -n $GATE_DEBUG && $GATE_DEBUG = "1" ]]; then
+  set -x
+fi
+
+set -e
+
+# Define NUM_NODES to appropriate values to check this number of k8s nodes.
+
+function upload_script() {
+  source "$GATE_UTILS"
+  BASENAME=$(basename $BASH_SOURCE)
+  # Copies script to genesis VM
+  rsync_cmd "$BASH_SOURCE" "$GENESIS_NAME:/root/airship/"
+  set -o pipefail
+  ssh_cmd_raw "$GENESIS_NAME" "GATE_DEBUG=${GATE_DEBUG} NUM_NODES=$1 /root/airship/${BASENAME}" 2>&1 | tee -a "$LOG_FILE"
+  set +o pipefail
+}
+
+function kubectl_retry() {
+  cnt=0
+  while true; do
+    "$KUBECTL" "$@"
+    ret=$?
+    cnt=$((cnt+1))
+    if [[ "$ret" -ne "0" ]]; then
+      if [[ "$cnt" -lt "$MAX_TRIES" ]]; then
+        sleep $PAUSE
+      else
+        return 1
+      fi
+    else
+      break
+    fi
+  done
+}
+
+function check_kube_nodes() {
+  try=0
+  while true; do
+    nodes_list=$(kubectl_retry get nodes --no-headers) || true
+    ret=$?
+    try=$((try+1))
+    if [ "$ret" -ne "0" ]; then
+      if [[ "$try" -lt "$MAX_TRIES" ]]; then
+        sleep $PAUSE
+      else
+        echo -e "Can't get nodes"
+        return 1
+      fi
+    fi
+    nodes_list="${nodes_list}\n"
+    all_nodes=$(echo -en "$nodes_list" | wc -l)
+    ok_nodes=$(echo -en "$nodes_list" | grep -c -v "NotReady" || true)
+    if [[ "$all_nodes" == "$1" && "$ok_nodes" == "$1" ]]; then
+      echo "Nodes: ${all_nodes}"
+      break
+    else
+      echo "Error: not enough nodes(found ${all_nodes}, ready ${ok_nodes}, expected $1)"
+      return 1
+    fi
+  done
+}
+
+function check_kube_components() {
+  try=0
+  while true; do
+    res=$(kubectl_retry get cs -o jsonpath="{.items[*].conditions[?(@.type == \"Healthy\")].status}") || true
+    try=$((try+1))
+
+    if $(echo $res | grep -q False); then
+      if [[ "$try" -lt "$MAX_TRIES" ]]; then
+        sleep $PAUSE
+      else
+        echo "Error: kubernetes components are not working properly"
+        kubectl_retry get cs
+        exit 1
+      fi
+    else
+      break
+    fi
+  done
+}
+
+if [[ -n "$GATE_UTILS" ]]; then
+  upload_script $NUM_NODES
+else
+set +e
+  KUBECTL="${KUBECTL:-/usr/local/bin/kubectl}"
+  NUM_NODES="${NUM_NODES:-4}"
+  PAUSE="${PAUSE:-1}"
+  MAX_TRIES="${MAX_TRIES:-3}"
+  if [[ ! -f "$KUBECTL" ]]; then
+    echo "Error: ${KUBECTL} not found"
+    exit 1
+  fi
+  exit_code=0
+  check_kube_nodes $NUM_NODES
+  nodes_status=$?
+  check_kube_components
+  components_status=$?
+  if [[ "$nodes_status" -ne "0" || "$components_status" -ne "0" ]]; then
+    echo "Kubernetes validation failed"
+    exit 1
+  else
+    echo "Kubernetes validation succeeded"
+  fi
+fi