From 165ccb2d7aae07c5ea0d9b5d46117a41b06d8ffb Mon Sep 17 00:00:00 2001 From: Ruslan Aliev Date: Wed, 9 Sep 2020 13:09:09 -0500 Subject: [PATCH] Add node availability check for deploy initinfra script Deploy initinfra script can fail due to "node01 not found error". We need to wait until node01 become ready and available before performing manipulations. Around 10% of gate script runner test fails due to this error. Example of error can be found here[1], example of handling - here[2]. [1] https://zuul.opendev.org/t/openstack/build/5f789f5b6f7c42af903a1892cc69ebf8 [2] https://zuul.opendev.org/t/openstack/build/be22de82d32d4c79a9f993d73fcd6d58/console Change-Id: If525042ec1260afae278542b033abb14b97347f6 Signed-off-by: Ruslan Aliev --- .../31_deploy_initinfra_target_node.sh | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tools/deployment/31_deploy_initinfra_target_node.sh b/tools/deployment/31_deploy_initinfra_target_node.sh index 638503a9b..5a0b3823f 100755 --- a/tools/deployment/31_deploy_initinfra_target_node.sh +++ b/tools/deployment/31_deploy_initinfra_target_node.sh @@ -15,14 +15,33 @@ set -xe export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"} +export TIMEOUT=${TIMEOUT:-60} +NODENAME="node01" # TODO need to run another config command after use-context to update kubeconfig echo "Switch context to target cluster and set manifest" airshipctl config use-context target-cluster-admin@target-cluster airshipctl config set-context target-cluster-admin@target-cluster --manifest dummy_manifest +end=$(($(date +%s) + $TIMEOUT)) +echo "Waiting $TIMEOUT seconds for $NODENAME to be created." +while true; do + if (kubectl --request-timeout 10s --kubeconfig $KUBECONFIG get nodes | grep -q $NODENAME) ; then + echo -e "\n$NODENAME found" + break + else + now=$(date +%s) + if [ $now -gt $end ]; then + echo -e "\n$NODENAME was not ready before TIMEOUT." + exit 1 + fi + echo -n . + sleep 10 + fi +done + # TODO remove taint -kubectl --kubeconfig $KUBECONFIG taint node node01 node-role.kubernetes.io/master- +kubectl --kubeconfig $KUBECONFIG taint node $NODENAME node-role.kubernetes.io/master- echo "Deploy infra to cluster" airshipctl phase apply initinfra --debug --wait-timeout 1000s