openstack/kayobe
Code Issues Proposed changes

CI: increase Ansible Galaxy retries & add delay

Browse Source 
We still see flakiness when downloading content from Ansible Galaxy,
often HTTP 520. This change increases the retries from 3 to 10, and adds
a 5 second delay between attempts.

Change-Id: I0c46e5fcc6979027dc6f1bc5cc49e923a205f654
Related: https://github.com/ansible/galaxy/issues/2429
This commit is contained in:
Mark Goddard 2021-03-26 17:24:05 +00:00
parent 775620733b
commit df00ba22e7
2 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
dev/functions
View File

@ -225,13 +225,16 @@ function run_kayobe {
} }
function control_host_bootstrap { function control_host_bootstrap {
attempts=10
interval=5
echo "Bootstrapping the Ansible control host" echo "Bootstrapping the Ansible control host"
for i in $(seq 1 3); do for i in $(seq 1 $attempts); do
if run_kayobe control host bootstrap; then if run_kayobe control host bootstrap; then
chb_success=1 chb_success=1
break break
fi fi
echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying" echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Sleeping $interval seconds before retrying"
sleep $interval
done done
if [[ -z ${chb_success+x} ]]; then if [[ -z ${chb_success+x} ]]; then
die $LINENO "Failed to bootstrap control host" die $LINENO "Failed to bootstrap control host"
@ -241,13 +244,16 @@ function control_host_bootstrap {
} }
function control_host_upgrade { function control_host_upgrade {
attempts=10
interval=5
echo "Upgrading the Ansible control host" echo "Upgrading the Ansible control host"
for i in $(seq 1 3); do for i in $(seq 1 $attempts); do
if run_kayobe control host upgrade; then if run_kayobe control host upgrade; then
chu_success=1 chu_success=1
break break
fi fi
echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying" echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Sleeping $interval seconds before retrying"
sleep $interval
done done
if [[ -z ${chu_success+x} ]]; then if [[ -z ${chu_success+x} ]]; then
die $LINENO "Failed to upgrade control host" die $LINENO "Failed to upgrade control host"

6
tools/ansible-galaxy-retried.sh
View File

@ -2,13 +2,15 @@
set -e set -e
GALAXY_RETRIES=${GALAXY_RETRIES:-3} GALAXY_RETRIES=${GALAXY_RETRIES:-10}
GALAXY_INTERVAL=${GALAXY_INTERVAL:-5}
for i in $(seq 1 $GALAXY_RETRIES); do for i in $(seq 1 $GALAXY_RETRIES); do
if ansible-galaxy "${@}"; then if ansible-galaxy "${@}"; then
exit 0 exit 0
fi fi
echo "Ansible Galaxy command failed. Retrying" echo "Ansible Galaxy command failed. Sleeping $GALAXY_INTERVAL seconds before retry"
sleep $GALAXY_INTERVAL
done done
echo "Failed to execute: ansible-galaxy ${@}" echo "Failed to execute: ansible-galaxy ${@}"