Browse Source

Script to smoke check kube

1. Export NUM_NODES(by default it will check 4 nodes in cluster).
2. Run as other stages.

Change-Id: I31fb525440afe5625445a410824d6a64cb32850e
Egorov, Stanislav (se6518) 3 months ago
parent
commit
9489f76255

+ 4
- 0
tools/multi_nodes_gate/airship_gate/lib/ssh.sh View File

@@ -2,6 +2,10 @@ rsync_cmd() {
2 2
     rsync -e "ssh -F ${SSH_CONFIG_DIR}/config" "${@}"
3 3
 }
4 4
 
5
+ssh_cmd_raw() {
6
+    ssh -F "${SSH_CONFIG_DIR}/config" $@
7
+}
8
+
5 9
 ssh_cmd() {
6 10
     HOST=${1}
7 11
     shift

+ 5
- 0
tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json View File

@@ -53,6 +53,11 @@
53 53
     {
54 54
       "name": "Deploy Site",
55 55
       "script": "shipyard-deploy-site.sh"
56
+    },
57
+    {
58
+      "name": "Validate Kube",
59
+      "script": "validate-kube.sh",
60
+      "on_error": "collect_genesis_info.sh"
56 61
     }
57 62
   ],
58 63
   "vm": {

+ 121
- 0
tools/multi_nodes_gate/airship_gate/stages/validate-kube.sh View File

@@ -0,0 +1,121 @@
1
+#!/usr/bin/env bash
2
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License");
5
+# you may not use this file except in compliance with the License.
6
+# You may obtain a copy of the License at
7
+#
8
+#     http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS,
12
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+# See the License for the specific language governing permissions and
14
+# limitations under the License.
15
+if [[ -n $GATE_DEBUG && $GATE_DEBUG = "1" ]]; then
16
+  set -x
17
+fi
18
+
19
+set -e
20
+
21
+# Define NUM_NODES to appropriate values to check this number of k8s nodes.
22
+
23
+function upload_script() {
24
+  source "$GATE_UTILS"
25
+  BASENAME=$(basename $BASH_SOURCE)
26
+  # Copies script to genesis VM
27
+  rsync_cmd "$BASH_SOURCE" "$GENESIS_NAME:/root/airship/"
28
+  set -o pipefail
29
+  ssh_cmd_raw "$GENESIS_NAME" "GATE_DEBUG=${GATE_DEBUG} NUM_NODES=$1 /root/airship/${BASENAME}" 2>&1 | tee -a "$LOG_FILE"
30
+  set +o pipefail
31
+}
32
+
33
+function kubectl_retry() {
34
+  cnt=0
35
+  while true; do
36
+    "$KUBECTL" "$@"
37
+    ret=$?
38
+    cnt=$((cnt+1))
39
+    if [[ "$ret" -ne "0" ]]; then
40
+      if [[ "$cnt" -lt "$MAX_TRIES" ]]; then
41
+        sleep $PAUSE
42
+      else
43
+        return 1
44
+      fi
45
+    else
46
+      break
47
+    fi
48
+  done
49
+}
50
+
51
+function check_kube_nodes() {
52
+  try=0
53
+  while true; do
54
+    nodes_list=$(kubectl_retry get nodes --no-headers) || true
55
+    ret=$?
56
+    try=$((try+1))
57
+    if [ "$ret" -ne "0" ]; then
58
+      if [[ "$try" -lt "$MAX_TRIES" ]]; then
59
+        sleep $PAUSE
60
+      else
61
+        echo -e "Can't get nodes"
62
+        return 1
63
+      fi
64
+    fi
65
+    nodes_list="${nodes_list}\n"
66
+    all_nodes=$(echo -en "$nodes_list" | wc -l)
67
+    ok_nodes=$(echo -en "$nodes_list" | grep -c -v "NotReady" || true)
68
+    if [[ "$all_nodes" == "$1" && "$ok_nodes" == "$1" ]]; then
69
+      echo "Nodes: ${all_nodes}"
70
+      break
71
+    else
72
+      echo "Error: not enough nodes(found ${all_nodes}, ready ${ok_nodes}, expected $1)"
73
+      return 1
74
+    fi
75
+  done
76
+}
77
+
78
+function check_kube_components() {
79
+  try=0
80
+  while true; do
81
+    res=$(kubectl_retry get cs -o jsonpath="{.items[*].conditions[?(@.type == \"Healthy\")].status}") || true
82
+    try=$((try+1))
83
+
84
+    if $(echo $res | grep -q False); then
85
+      if [[ "$try" -lt "$MAX_TRIES" ]]; then
86
+        sleep $PAUSE
87
+      else
88
+        echo "Error: kubernetes components are not working properly"
89
+        kubectl_retry get cs
90
+        exit 1
91
+      fi
92
+    else
93
+      break
94
+    fi
95
+  done
96
+}
97
+
98
+if [[ -n "$GATE_UTILS" ]]; then
99
+  upload_script $NUM_NODES
100
+else
101
+set +e
102
+  KUBECTL="${KUBECTL:-/usr/local/bin/kubectl}"
103
+  NUM_NODES="${NUM_NODES:-4}"
104
+  PAUSE="${PAUSE:-1}"
105
+  MAX_TRIES="${MAX_TRIES:-3}"
106
+  if [[ ! -f "$KUBECTL" ]]; then
107
+    echo "Error: ${KUBECTL} not found"
108
+    exit 1
109
+  fi
110
+  exit_code=0
111
+  check_kube_nodes $NUM_NODES
112
+  nodes_status=$?
113
+  check_kube_components
114
+  components_status=$?
115
+  if [[ "$nodes_status" -ne "0" || "$components_status" -ne "0" ]]; then
116
+    echo "Kubernetes validation failed"
117
+    exit 1
118
+  else
119
+    echo "Kubernetes validation succeeded"
120
+  fi
121
+fi

Loading…
Cancel
Save