From 7754441df16eaf43ba026636f4076e2cbcf8f493 Mon Sep 17 00:00:00 2001
From: Ahmad Mahmoudi <ahmad.mahmoudi@att.com>
Date: Mon, 4 May 2020 17:08:34 +0000
Subject: [PATCH] (fix) Added retries for genesis.sh

Added retries to rsync the genesis.sh to the genesis node, in case genesis
is rebooted as part of pre-genesis stage. This fix waits and retries until
genesis node is reachab, during reboot or for any other reason.

Added wait in genesis.sh to wait for genesis node to come up, in case it
was rebooted in the previous stage.

Change-Id: I538f2c7b1543e6775ad580ccd3dc0b5cc88d68b1
---
 .../seaworthy-virt/airship_gate/stages/genesis.sh  | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tools/deployment/seaworthy-virt/airship_gate/stages/genesis.sh b/tools/deployment/seaworthy-virt/airship_gate/stages/genesis.sh
index c78a68924..af64f68b7 100755
--- a/tools/deployment/seaworthy-virt/airship_gate/stages/genesis.sh
+++ b/tools/deployment/seaworthy-virt/airship_gate/stages/genesis.sh
@@ -18,7 +18,19 @@ set -e
 source "${GATE_UTILS}"
 
 # Copies script and virtmgr private key to genesis VM
-rsync_cmd "${SCRIPT_DEPOT}/genesis.sh" "${GENESIS_NAME}:/root/airship/"
+# waits for the genesis node to complete reboot, if it is rebooted during
+# genesis setup stage.
+#rsync_cmd "${SCRIPT_DEPOT}/genesis.sh" "${GENESIS_NAME}:/root/airship/"
+retries=5
+while ! rsync_cmd "${SCRIPT_DEPOT}/genesis.sh" "${GENESIS_NAME}:/root/airship/"; do
+  retries=$((retries-1))
+  if [[ "${reties}" == "0" ]]; then
+    log_error "Genesis is not reachable after ${retries} retries."
+    break
+  fi
+  log "Genesis node is not reachable yet. Retrying in 30 seconds."
+  sleep 30
+done
 
 set -o pipefail
 ssh_cmd_raw "${GENESIS_NAME}" "PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY} /root/airship/genesis.sh" 2>&1 | tee -a "${LOG_FILE}"