From eccb119877013b68f41d987b8822bc1ccee51ed8 Mon Sep 17 00:00:00 2001
From: Davlet Panech <davlet.panech@windriver.com>
Date: Thu, 25 May 2023 15:45:47 -0400
Subject: [PATCH] build-docker-images: add retry delay

Add retry delay to wheels and docker image build scripts.

TESTS
=================================
* Build docker base image, wheels and one docker image
* Check Jenkins logs and make sure scripts are called with --attempts
  and --retry-delay

Story: 2010055
Task:  48106

Depends-On: https://review.opendev.org/c/starlingx/root/+/877119
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
Change-Id: I771e092ef340db8e639c121c0ebc990790eb0568
---
 scripts/build-docker-base.sh            | 4 +++-
 scripts/build-docker-images.sh          | 4 +++-
 scripts/build-wheels.sh                 | 1 +
 scripts/lib/job_utils.sh                | 2 ++
 scripts/templates/build.conf.example.in | 3 +++
 5 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/scripts/build-docker-base.sh b/scripts/build-docker-base.sh
index ab7b0af..307c9e6 100755
--- a/scripts/build-docker-base.sh
+++ b/scripts/build-docker-base.sh
@@ -8,6 +8,7 @@
 
 set -e
 source $(dirname "$0")/lib/job_utils.sh
+source $(dirname "$0")/lib/retries.sh
 
 require_job_env BUILD_HOME
 require_job_env DRY_RUN
@@ -37,6 +38,7 @@ declare -a cmd=(
     "--os=$DOCKER_BASE_OS"
     "--version=$base_image_tag"
     "--attempts=$DOCKER_BUILD_RETRY_COUNT"
+    "--retry-delay=$DOCKER_BUILD_RETRY_DELAY"
     "--stream=$BUILD_STREAM"
     "--user=$DOCKER_REGISTRY_ORG"
     "--latest"
@@ -72,7 +74,7 @@ stx_docker_cmd $DRY_RUN_ARG "cd \$MY_REPO/build-tools/build-docker-images && ${c
 if $PUSH_DOCKER_IMAGES ; then
     for reg in $EXTRA_REGISTRY_PREFIX_LIST ; do
         stx_docker_cmd $DRY_RUN_ARG "docker tag $DOCKER_REGISTRY/$DOCKER_REGISTRY_ORG/stx-$DOCKER_BASE_OS:$base_image_tag $reg/stx-$DOCKER_BASE_OS:$base_image_tag"
-        stx_docker_cmd $DRY_RUN_ARG "docker push $reg/stx-$DOCKER_BASE_OS:$base_image_tag"
+        with_retries -d "$DOCKER_BUILD_RETRY_DELAY" "$DOCKER_BUILD_RETRY_COUNT" stx_docker_cmd $DRY_RUN_ARG "docker push $reg/stx-$DOCKER_BASE_OS:$base_image_tag"
     done
 fi
 
diff --git a/scripts/build-docker-images.sh b/scripts/build-docker-images.sh
index 12b000f..2c4d553 100755
--- a/scripts/build-docker-images.sh
+++ b/scripts/build-docker-images.sh
@@ -8,6 +8,7 @@
 
 set -e
 source $(dirname "$0")/lib/job_utils.sh
+source $(dirname "$0")/lib/retries.sh
 
 require_job_env BUILD_HOME
 require_job_env DRY_RUN
@@ -39,6 +40,7 @@ declare -a cmd=(
     "./build-stx-images.sh"
     "--os-label=$DOCKER_BASE_OS"
     "--attempts=$DOCKER_BUILD_RETRY_COUNT"
+    "--retry-delay=$DOCKER_BUILD_RETRY_DELAY"
     "--stream=$BUILD_STREAM"
     "--version=$TIMESTAMP"
     "--prefix=${BUILD_BRANCH_FOR_IMAGE_TAG}"
@@ -105,7 +107,7 @@ retag_and_push() {
             for reg_prefix in $EXTRA_REGISTRY_PREFIX_LIST ; do
                 local dst_img="$(echo "${reg_prefix}/$base_img" | sed 's!//*!/!g')"
                 stx_docker_cmd $DRY_RUN_ARG "docker tag $src_img $dst_img"
-                stx_docker_cmd $DRY_RUN_ARG "docker push $dst_img"
+                with_retries -d "$DOCKER_BUILD_RETRY_DELAY" "$DOCKER_BUILD_RETRY_COUNT" stx_docker_cmd $DRY_RUN_ARG "docker push $dst_img"
             done
         done
     fi
diff --git a/scripts/build-wheels.sh b/scripts/build-wheels.sh
index 73c9e56..7a11fdc 100755
--- a/scripts/build-wheels.sh
+++ b/scripts/build-wheels.sh
@@ -80,6 +80,7 @@ cmd=(
     "--os=$DOCKER_BASE_OS"
     "--stream=$BUILD_STREAM"
     "--attempts=$DOCKER_BUILD_RETRY_COUNT"
+    "--retry-delay=$DOCKER_BUILD_RETRY_DELAY"
 )
 
 if [[ "$USE_DOCKER_CACHE" == true ]] ; then
diff --git a/scripts/lib/job_utils.sh b/scripts/lib/job_utils.sh
index 97daf83..b16c792 100644
--- a/scripts/lib/job_utils.sh
+++ b/scripts/lib/job_utils.sh
@@ -217,6 +217,8 @@ __set_build_vars() {
 
     declare_job_env SIGN_MAX_ATTEMPTS 3
     declare_job_env SIGN_BACKOFF_DELAY 10
+    declare_job_env DOCKER_BUILD_RETRY_COUNT 3
+    declare_job_env DOCKER_BUILD_RETRY_DELAY 30
 
 }
 
diff --git a/scripts/templates/build.conf.example.in b/scripts/templates/build.conf.example.in
index 7dc2ce6..8894b0d 100644
--- a/scripts/templates/build.conf.example.in
+++ b/scripts/templates/build.conf.example.in
@@ -147,6 +147,9 @@ PUBLISH_LATEST_LINK=false          # create latest symlink?
 # How many times to try building docker images
 DOCKER_BUILD_RETRY_COUNT=3
 
+# Sleep between retries (seconds)
+DOCKER_BUILD_RETRY_DELAY=30
+
 # Docker config file used for authentication when pushing images. If empty,
 # defaults to $USER/.docker/config.json. May be relative to $BUILD_HOME.
 DOCKER_CONFIG_FILE="docker-config.json"