airship/treasuremap
Code Issues Proposed changes

Merge "(fix) Added retries for genesis deploy"

Browse Source
This commit is contained in:
Zuul 2020-08-04 17:58:36 +00:00 committed by Gerrit Code Review
commit fbae9b7bdf
3 changed files with 35 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tools/deployment/seaworthy-virt/airship_gate/lib/airship.sh
View File

@ -133,6 +133,7 @@ shipyard_action_wait() {
exit 0
fi
log "$(shipyard_cmd describe "${ACTION_ID}")"
sleep "${poll_time}"
done
}

19
tools/deployment/seaworthy-virt/airship_gate/stages/genesis.sh
View File

@ -18,13 +18,26 @@ set -e
source "${GATE_UTILS}"
# Copies script and virtmgr private key to genesis VM
rsync_cmd "${SCRIPT_DEPOT}/genesis.sh" "${GENESIS_NAME}:/root/airship/"
# waits for the genesis node to complete reboot, if it is rebooted during
# genesis setup stage.
GENESIS_RSYNC_RETRIES=${GENESIS_RSYNC_RETRIES:-10}
retries=0
while ! rsync_cmd "${SCRIPT_DEPOT}/genesis.sh" "${GENESIS_NAME}:/root/airship/"; do
if [[ "${retries}" < "${GENESIS_RSYNC_RETRIES}" ]]; then
log "Genesis node is not reachable yet. Retrying in 30 seconds."
retries=$((retries+1))
sleep 30
continue
fi
log_error "Genesis was not reachable after max retries: "${GENESIS_RSYNC_RETRIES}"."
exit 1
done
set -o pipefail
ssh_cmd_raw "${GENESIS_NAME}" "PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY} /root/airship/genesis.sh" 2>&1 | tee -a "${LOG_FILE}"
set +o pipefail
if ! ssh_cmd n0 docker images | tail -n +2 | grep -v registry:5000 ; then
if ! ssh_cmd "${GENESIS_NAME}" docker images | tail -n +2 | grep -v registry:5000 ; then
log_warn "Using some non-cached docker images. This will slow testing."
ssh_cmd n0 docker images | tail -n +2 | grep -v registry:5000 | tee -a "${LOG_FILE}"
ssh_cmd "${GENESIS_NAME}" docker images | tail -n +2 | grep -v registry:5000 | tee -a "${LOG_FILE}"
fi

23
tools/deployment/seaworthy-virt/airship_gate/stages/shipyard-load-design.sh
View File

@ -52,12 +52,25 @@ check_configdocs_result(){
fi
}
CREATE_CONFIGDOCS_RETRIES=${CREATE_CONFIGDOCS_RETRIES:-5}
create_configdocs_design() {
for ((i=0; i<${CREATE_CONFIGDOCS_RETRIES}; i++)); do
if check_configdocs_result "$(shipyard_cmd create configdocs "$@")"; then
log "Create confidocs succeeded."
return 0
fi
log "Failed on atemp $i, retrying ..."
sleep 30
done
log "Create configdocs failed after $i retries."
return 1
}
# Copy site design to genesis node
ssh_cmd "${BUILD_NAME}" mkdir -p "${BUILD_WORK_DIR}/site"
rsync_cmd "${DEFINITION_DEPOT}"/*.yaml "${BUILD_NAME}:${BUILD_WORK_DIR}/site/"
sleep 120
check_configdocs_result "$(shipyard_cmd create configdocs design "--directory=${BUILD_WORK_DIR}/site" --replace)"
create_configdocs_design design --directory="${BUILD_WORK_DIR}/site" --replace
# Skip certs/gate if already part of site manifests
if [[ -n "${USE_EXISTING_SECRETS}" ]]
@ -70,14 +83,14 @@ if [[ "${OMIT_CERTS}" == "0" ]]
then
ssh_cmd "${BUILD_NAME}" mkdir -p "${BUILD_WORK_DIR}/certs"
rsync_cmd "${CERT_DEPOT}"/*.yaml "${BUILD_NAME}:${BUILD_WORK_DIR}/certs/"
check_configdocs_result "$(shipyard_cmd create configdocs certs "--directory=${BUILD_WORK_DIR}/certs" --append)"
create_configdocs_design certs --directory="${BUILD_WORK_DIR}/certs" --append
fi
if [[ "${OMIT_GATE}" == "0" ]]
then
ssh_cmd "${BUILD_NAME}" mkdir -p "${BUILD_WORK_DIR}/gate"
rsync_cmd "${GATE_DEPOT}"/*.yaml "${BUILD_NAME}:${BUILD_WORK_DIR}/gate/"
check_configdocs_result "$(shipyard_cmd create configdocs gate "--directory=${BUILD_WORK_DIR}/gate" --append)"
create_configdocs_design gate --directory="${BUILD_WORK_DIR}/gate" --append
fi
check_configdocs_result "$(shipyard_cmd commit configdocs)"