diff --git a/tools/deployment/seaworthy-virt/airship_gate/lib/airship.sh b/tools/deployment/seaworthy-virt/airship_gate/lib/airship.sh index 26f4dbb78..7b4f1b628 100644 --- a/tools/deployment/seaworthy-virt/airship_gate/lib/airship.sh +++ b/tools/deployment/seaworthy-virt/airship_gate/lib/airship.sh @@ -133,6 +133,7 @@ shipyard_action_wait() { exit 0 fi + log "$(shipyard_cmd describe "${ACTION_ID}")" sleep "${poll_time}" done } diff --git a/tools/deployment/seaworthy-virt/airship_gate/stages/genesis.sh b/tools/deployment/seaworthy-virt/airship_gate/stages/genesis.sh index c78a68924..b08c1595a 100755 --- a/tools/deployment/seaworthy-virt/airship_gate/stages/genesis.sh +++ b/tools/deployment/seaworthy-virt/airship_gate/stages/genesis.sh @@ -18,13 +18,26 @@ set -e source "${GATE_UTILS}" # Copies script and virtmgr private key to genesis VM -rsync_cmd "${SCRIPT_DEPOT}/genesis.sh" "${GENESIS_NAME}:/root/airship/" +# waits for the genesis node to complete reboot, if it is rebooted during +# genesis setup stage. +GENESIS_RSYNC_RETRIES=${GENESIS_RSYNC_RETRIES:-10} +retries=0 +while ! rsync_cmd "${SCRIPT_DEPOT}/genesis.sh" "${GENESIS_NAME}:/root/airship/"; do + if [[ "${retries}" < "${GENESIS_RSYNC_RETRIES}" ]]; then + log "Genesis node is not reachable yet. Retrying in 30 seconds." + retries=$((retries+1)) + sleep 30 + continue + fi + log_error "Genesis was not reachable after max retries: "${GENESIS_RSYNC_RETRIES}"." + exit 1 +done set -o pipefail ssh_cmd_raw "${GENESIS_NAME}" "PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY} /root/airship/genesis.sh" 2>&1 | tee -a "${LOG_FILE}" set +o pipefail -if ! ssh_cmd n0 docker images | tail -n +2 | grep -v registry:5000 ; then +if ! ssh_cmd "${GENESIS_NAME}" docker images | tail -n +2 | grep -v registry:5000 ; then log_warn "Using some non-cached docker images. This will slow testing." - ssh_cmd n0 docker images | tail -n +2 | grep -v registry:5000 | tee -a "${LOG_FILE}" + ssh_cmd "${GENESIS_NAME}" docker images | tail -n +2 | grep -v registry:5000 | tee -a "${LOG_FILE}" fi diff --git a/tools/deployment/seaworthy-virt/airship_gate/stages/shipyard-load-design.sh b/tools/deployment/seaworthy-virt/airship_gate/stages/shipyard-load-design.sh index f957a010a..6ddd65ece 100755 --- a/tools/deployment/seaworthy-virt/airship_gate/stages/shipyard-load-design.sh +++ b/tools/deployment/seaworthy-virt/airship_gate/stages/shipyard-load-design.sh @@ -52,12 +52,25 @@ check_configdocs_result(){ fi } +CREATE_CONFIGDOCS_RETRIES=${CREATE_CONFIGDOCS_RETRIES:-5} + +create_configdocs_design() { + for ((i=0; i<${CREATE_CONFIGDOCS_RETRIES}; i++)); do + if check_configdocs_result "$(shipyard_cmd create configdocs "$@")"; then + log "Create confidocs succeeded." + return 0 + fi + log "Failed on atemp $i, retrying ..." + sleep 30 + done + log "Create configdocs failed after $i retries." + return 1 +} + # Copy site design to genesis node ssh_cmd "${BUILD_NAME}" mkdir -p "${BUILD_WORK_DIR}/site" rsync_cmd "${DEFINITION_DEPOT}"/*.yaml "${BUILD_NAME}:${BUILD_WORK_DIR}/site/" - -sleep 120 -check_configdocs_result "$(shipyard_cmd create configdocs design "--directory=${BUILD_WORK_DIR}/site" --replace)" +create_configdocs_design design --directory="${BUILD_WORK_DIR}/site" --replace # Skip certs/gate if already part of site manifests if [[ -n "${USE_EXISTING_SECRETS}" ]] @@ -70,14 +83,14 @@ if [[ "${OMIT_CERTS}" == "0" ]] then ssh_cmd "${BUILD_NAME}" mkdir -p "${BUILD_WORK_DIR}/certs" rsync_cmd "${CERT_DEPOT}"/*.yaml "${BUILD_NAME}:${BUILD_WORK_DIR}/certs/" - check_configdocs_result "$(shipyard_cmd create configdocs certs "--directory=${BUILD_WORK_DIR}/certs" --append)" + create_configdocs_design certs --directory="${BUILD_WORK_DIR}/certs" --append fi if [[ "${OMIT_GATE}" == "0" ]] then ssh_cmd "${BUILD_NAME}" mkdir -p "${BUILD_WORK_DIR}/gate" rsync_cmd "${GATE_DEPOT}"/*.yaml "${BUILD_NAME}:${BUILD_WORK_DIR}/gate/" - check_configdocs_result "$(shipyard_cmd create configdocs gate "--directory=${BUILD_WORK_DIR}/gate" --append)" + create_configdocs_design gate --directory="${BUILD_WORK_DIR}/gate" --append fi check_configdocs_result "$(shipyard_cmd commit configdocs)"