Merge "Script to smoke check kube"
This commit is contained in:
commit
64a148d6ef
@ -2,6 +2,10 @@ rsync_cmd() {
|
|||||||
rsync -e "ssh -F ${SSH_CONFIG_DIR}/config" "${@}"
|
rsync -e "ssh -F ${SSH_CONFIG_DIR}/config" "${@}"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ssh_cmd_raw() {
|
||||||
|
ssh -F "${SSH_CONFIG_DIR}/config" $@
|
||||||
|
}
|
||||||
|
|
||||||
ssh_cmd() {
|
ssh_cmd() {
|
||||||
HOST=${1}
|
HOST=${1}
|
||||||
shift
|
shift
|
||||||
|
@ -53,6 +53,11 @@
|
|||||||
{
|
{
|
||||||
"name": "Deploy Site",
|
"name": "Deploy Site",
|
||||||
"script": "shipyard-deploy-site.sh"
|
"script": "shipyard-deploy-site.sh"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Validate Kube",
|
||||||
|
"script": "validate-kube.sh",
|
||||||
|
"on_error": "collect_genesis_info.sh"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"vm": {
|
"vm": {
|
||||||
|
121
tools/multi_nodes_gate/airship_gate/stages/validate-kube.sh
Executable file
121
tools/multi_nodes_gate/airship_gate/stages/validate-kube.sh
Executable file
@ -0,0 +1,121 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
if [[ -n $GATE_DEBUG && $GATE_DEBUG = "1" ]]; then
|
||||||
|
set -x
|
||||||
|
fi
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Define NUM_NODES to appropriate values to check this number of k8s nodes.
|
||||||
|
|
||||||
|
function upload_script() {
|
||||||
|
source "$GATE_UTILS"
|
||||||
|
BASENAME=$(basename $BASH_SOURCE)
|
||||||
|
# Copies script to genesis VM
|
||||||
|
rsync_cmd "$BASH_SOURCE" "$GENESIS_NAME:/root/airship/"
|
||||||
|
set -o pipefail
|
||||||
|
ssh_cmd_raw "$GENESIS_NAME" "GATE_DEBUG=${GATE_DEBUG} NUM_NODES=$1 /root/airship/${BASENAME}" 2>&1 | tee -a "$LOG_FILE"
|
||||||
|
set +o pipefail
|
||||||
|
}
|
||||||
|
|
||||||
|
function kubectl_retry() {
|
||||||
|
cnt=0
|
||||||
|
while true; do
|
||||||
|
"$KUBECTL" "$@"
|
||||||
|
ret=$?
|
||||||
|
cnt=$((cnt+1))
|
||||||
|
if [[ "$ret" -ne "0" ]]; then
|
||||||
|
if [[ "$cnt" -lt "$MAX_TRIES" ]]; then
|
||||||
|
sleep $PAUSE
|
||||||
|
else
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
function check_kube_nodes() {
|
||||||
|
try=0
|
||||||
|
while true; do
|
||||||
|
nodes_list=$(kubectl_retry get nodes --no-headers) || true
|
||||||
|
ret=$?
|
||||||
|
try=$((try+1))
|
||||||
|
if [ "$ret" -ne "0" ]; then
|
||||||
|
if [[ "$try" -lt "$MAX_TRIES" ]]; then
|
||||||
|
sleep $PAUSE
|
||||||
|
else
|
||||||
|
echo -e "Can't get nodes"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
nodes_list="${nodes_list}\n"
|
||||||
|
all_nodes=$(echo -en "$nodes_list" | wc -l)
|
||||||
|
ok_nodes=$(echo -en "$nodes_list" | grep -c -v "NotReady" || true)
|
||||||
|
if [[ "$all_nodes" == "$1" && "$ok_nodes" == "$1" ]]; then
|
||||||
|
echo "Nodes: ${all_nodes}"
|
||||||
|
break
|
||||||
|
else
|
||||||
|
echo "Error: not enough nodes(found ${all_nodes}, ready ${ok_nodes}, expected $1)"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
function check_kube_components() {
|
||||||
|
try=0
|
||||||
|
while true; do
|
||||||
|
res=$(kubectl_retry get cs -o jsonpath="{.items[*].conditions[?(@.type == \"Healthy\")].status}") || true
|
||||||
|
try=$((try+1))
|
||||||
|
|
||||||
|
if $(echo $res | grep -q False); then
|
||||||
|
if [[ "$try" -lt "$MAX_TRIES" ]]; then
|
||||||
|
sleep $PAUSE
|
||||||
|
else
|
||||||
|
echo "Error: kubernetes components are not working properly"
|
||||||
|
kubectl_retry get cs
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ -n "$GATE_UTILS" ]]; then
|
||||||
|
upload_script $NUM_NODES
|
||||||
|
else
|
||||||
|
set +e
|
||||||
|
KUBECTL="${KUBECTL:-/usr/local/bin/kubectl}"
|
||||||
|
NUM_NODES="${NUM_NODES:-4}"
|
||||||
|
PAUSE="${PAUSE:-1}"
|
||||||
|
MAX_TRIES="${MAX_TRIES:-3}"
|
||||||
|
if [[ ! -f "$KUBECTL" ]]; then
|
||||||
|
echo "Error: ${KUBECTL} not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
exit_code=0
|
||||||
|
check_kube_nodes $NUM_NODES
|
||||||
|
nodes_status=$?
|
||||||
|
check_kube_components
|
||||||
|
components_status=$?
|
||||||
|
if [[ "$nodes_status" -ne "0" || "$components_status" -ne "0" ]]; then
|
||||||
|
echo "Kubernetes validation failed"
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
echo "Kubernetes validation succeeded"
|
||||||
|
fi
|
||||||
|
fi
|
Loading…
Reference in New Issue
Block a user