diff --git a/dev/functions b/dev/functions index 019c54fd1..00377528d 100644 --- a/dev/functions +++ b/dev/functions @@ -225,13 +225,16 @@ function run_kayobe { } function control_host_bootstrap { + attempts=10 + interval=5 echo "Bootstrapping the Ansible control host" - for i in $(seq 1 3); do + for i in $(seq 1 $attempts); do if run_kayobe control host bootstrap; then chb_success=1 break fi - echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying" + echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Sleeping $interval seconds before retrying" + sleep $interval done if [[ -z ${chb_success+x} ]]; then die $LINENO "Failed to bootstrap control host" @@ -241,13 +244,16 @@ function control_host_bootstrap { } function control_host_upgrade { + attempts=10 + interval=5 echo "Upgrading the Ansible control host" - for i in $(seq 1 3); do + for i in $(seq 1 $attempts); do if run_kayobe control host upgrade; then chu_success=1 break fi - echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying" + echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Sleeping $interval seconds before retrying" + sleep $interval done if [[ -z ${chu_success+x} ]]; then die $LINENO "Failed to upgrade control host" diff --git a/tools/ansible-galaxy-retried.sh b/tools/ansible-galaxy-retried.sh index 4082f98e2..846c132f9 100755 --- a/tools/ansible-galaxy-retried.sh +++ b/tools/ansible-galaxy-retried.sh @@ -2,13 +2,15 @@ set -e -GALAXY_RETRIES=${GALAXY_RETRIES:-3} +GALAXY_RETRIES=${GALAXY_RETRIES:-10} +GALAXY_INTERVAL=${GALAXY_INTERVAL:-5} for i in $(seq 1 $GALAXY_RETRIES); do if ansible-galaxy "${@}"; then exit 0 fi - echo "Ansible Galaxy command failed. Retrying" + echo "Ansible Galaxy command failed. Sleeping $GALAXY_INTERVAL seconds before retry" + sleep $GALAXY_INTERVAL done echo "Failed to execute: ansible-galaxy ${@}"