diff --git a/scripts/build-docker-base.sh b/scripts/build-docker-base.sh index ab7b0af..307c9e6 100755 --- a/scripts/build-docker-base.sh +++ b/scripts/build-docker-base.sh @@ -8,6 +8,7 @@ set -e source $(dirname "$0")/lib/job_utils.sh +source $(dirname "$0")/lib/retries.sh require_job_env BUILD_HOME require_job_env DRY_RUN @@ -37,6 +38,7 @@ declare -a cmd=( "--os=$DOCKER_BASE_OS" "--version=$base_image_tag" "--attempts=$DOCKER_BUILD_RETRY_COUNT" + "--retry-delay=$DOCKER_BUILD_RETRY_DELAY" "--stream=$BUILD_STREAM" "--user=$DOCKER_REGISTRY_ORG" "--latest" @@ -72,7 +74,7 @@ stx_docker_cmd $DRY_RUN_ARG "cd \$MY_REPO/build-tools/build-docker-images && ${c if $PUSH_DOCKER_IMAGES ; then for reg in $EXTRA_REGISTRY_PREFIX_LIST ; do stx_docker_cmd $DRY_RUN_ARG "docker tag $DOCKER_REGISTRY/$DOCKER_REGISTRY_ORG/stx-$DOCKER_BASE_OS:$base_image_tag $reg/stx-$DOCKER_BASE_OS:$base_image_tag" - stx_docker_cmd $DRY_RUN_ARG "docker push $reg/stx-$DOCKER_BASE_OS:$base_image_tag" + with_retries -d "$DOCKER_BUILD_RETRY_DELAY" "$DOCKER_BUILD_RETRY_COUNT" stx_docker_cmd $DRY_RUN_ARG "docker push $reg/stx-$DOCKER_BASE_OS:$base_image_tag" done fi diff --git a/scripts/build-docker-images.sh b/scripts/build-docker-images.sh index 12b000f..2c4d553 100755 --- a/scripts/build-docker-images.sh +++ b/scripts/build-docker-images.sh @@ -8,6 +8,7 @@ set -e source $(dirname "$0")/lib/job_utils.sh +source $(dirname "$0")/lib/retries.sh require_job_env BUILD_HOME require_job_env DRY_RUN @@ -39,6 +40,7 @@ declare -a cmd=( "./build-stx-images.sh" "--os-label=$DOCKER_BASE_OS" "--attempts=$DOCKER_BUILD_RETRY_COUNT" + "--retry-delay=$DOCKER_BUILD_RETRY_DELAY" "--stream=$BUILD_STREAM" "--version=$TIMESTAMP" "--prefix=${BUILD_BRANCH_FOR_IMAGE_TAG}" @@ -105,7 +107,7 @@ retag_and_push() { for reg_prefix in $EXTRA_REGISTRY_PREFIX_LIST ; do local dst_img="$(echo "${reg_prefix}/$base_img" | sed 's!//*!/!g')" stx_docker_cmd $DRY_RUN_ARG "docker tag $src_img $dst_img" - stx_docker_cmd $DRY_RUN_ARG "docker push $dst_img" + with_retries -d "$DOCKER_BUILD_RETRY_DELAY" "$DOCKER_BUILD_RETRY_COUNT" stx_docker_cmd $DRY_RUN_ARG "docker push $dst_img" done done fi diff --git a/scripts/build-wheels.sh b/scripts/build-wheels.sh index 73c9e56..7a11fdc 100755 --- a/scripts/build-wheels.sh +++ b/scripts/build-wheels.sh @@ -80,6 +80,7 @@ cmd=( "--os=$DOCKER_BASE_OS" "--stream=$BUILD_STREAM" "--attempts=$DOCKER_BUILD_RETRY_COUNT" + "--retry-delay=$DOCKER_BUILD_RETRY_DELAY" ) if [[ "$USE_DOCKER_CACHE" == true ]] ; then diff --git a/scripts/lib/job_utils.sh b/scripts/lib/job_utils.sh index 97daf83..b16c792 100644 --- a/scripts/lib/job_utils.sh +++ b/scripts/lib/job_utils.sh @@ -217,6 +217,8 @@ __set_build_vars() { declare_job_env SIGN_MAX_ATTEMPTS 3 declare_job_env SIGN_BACKOFF_DELAY 10 + declare_job_env DOCKER_BUILD_RETRY_COUNT 3 + declare_job_env DOCKER_BUILD_RETRY_DELAY 30 } diff --git a/scripts/templates/build.conf.example.in b/scripts/templates/build.conf.example.in index 7dc2ce6..8894b0d 100644 --- a/scripts/templates/build.conf.example.in +++ b/scripts/templates/build.conf.example.in @@ -147,6 +147,9 @@ PUBLISH_LATEST_LINK=false # create latest symlink? # How many times to try building docker images DOCKER_BUILD_RETRY_COUNT=3 +# Sleep between retries (seconds) +DOCKER_BUILD_RETRY_DELAY=30 + # Docker config file used for authentication when pushing images. If empty, # defaults to $USER/.docker/config.json. May be relative to $BUILD_HOME. DOCKER_CONFIG_FILE="docker-config.json"