Retries for certain wheel and image build commands

The wheel and image build tools hit intermittent issues such as:
* Network or server issues that cause file download failures
* Repo access failures due to repo updates or other issues

To avoid formal build failures on such issues, this update adds
retries around specific commands, like wget and docker build.

Change-Id: Ifafdc6201872f43b2dd77efd4dcb033456477c2e
Closes-Bug: 1823986
Signed-off-by: Don Penney <don.penney@windriver.com>

build-tools/build-docker-images/build-stx-base.sh

@@ -1,14 +1,16 @@
 #!/bin/bash
 #
-# Copyright (c) 2018 Wind River Systems, Inc.
+# Copyright (c) 2018-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-# This utility builds the StarlingX wheel tarball
+# This utility builds the StarlingX base image
 #
 
 MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
 
+source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
+
 # Required env vars
 if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
     echo "Environment not setup for builds" >&2
@@ -31,6 +33,7 @@ CLEAN=no
 TAG_LATEST=no
 LATEST_TAG=latest
 HOST=${HOSTNAME}
+declare -i MAX_ATTEMPTS=1
 
 function usage {
     cat >&2 <<EOF
@@ -52,11 +55,12 @@ Options:
     --registry:   Docker registry
     --clean:      Remove image(s) from local registry
     --hostname:   build repo host
+    --attempts:   Max attempts, in case of failure (default: 1)
 
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname:,attempts: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -132,6 +136,10 @@ while true; do
             HOST=$2
             shift 2
             ;;
+        --attempts)
+            MAX_ATTEMPTS=$2
+            shift 2
+            ;;
         -h | --help )
             usage
             exit 1
@@ -223,6 +231,9 @@ done
 docker images --format '{{.Repository}}:{{.Tag}}' ${OS}:${OS_VERSION} | grep -q "^${OS}:${OS_VERSION}$"
 BASE_IMAGE_PRESENT=$?
 
+# Pull the image anyway, to ensure it is up to date
+docker pull ${OS}:${OS_VERSION}
+
 # Build the image
 IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION}
 IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG}
@@ -238,7 +249,7 @@ fi
 BUILD_ARGS+=(--tag ${IMAGE_NAME} ${BUILDDIR})
 
 # Build base image
-docker build "${BUILD_ARGS[@]}"
+with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
 if [ $? -ne 0 ]; then
     echo "Failed running docker build command" >&2
     exit 1

build-tools/build-docker-images/build-stx-images.sh

@@ -1,14 +1,16 @@
 #!/bin/bash
 #
-# Copyright (c) 2018 Wind River Systems, Inc.
+# Copyright (c) 2018-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-# This utility builds the StarlingX wheel tarball
+# This utility builds the StarlingX container images
 #
 
 MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
 
+source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
+
 # Required env vars
 if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
     echo "Environment not setup for builds" >&2
@@ -35,6 +37,7 @@ TAG_LIST_FILE=
 TAG_LIST_LATEST_FILE=
 declare -a ONLY
 declare -a SKIP
+declare -i MAX_ATTEMPTS=1
 
 function usage {
     cat >&2 <<EOF
@@ -61,6 +64,7 @@ Options:
     --skip <image> : Skip building the specified image(s). Multiple images
                      can be specified with a comma-separated list, or with
                      multiple --skip arguments.
+    --attempts:   Max attempts, in case of failure (default: 1)
 
 
 EOF
@@ -276,7 +280,7 @@ function build_image_loci {
 
     local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}"
 
-    docker build ${WORKDIR}/loci --no-cache \
+    with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci --no-cache \
         "${BUILD_ARGS[@]}" \
         --tag ${build_image_name}  2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
     if [ ${PIPESTATUS[0]} -ne 0 ]; then
@@ -361,7 +365,7 @@ function build_image_docker {
         BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY)
     fi
     BASE_BUILD_ARGS+=(--tag ${build_image_name})
-    docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
+    with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
 
     if [ ${PIPESTATUS[0]} -ne 0 ]; then
         echo "Failed to build ${LABEL}... Aborting"
@@ -395,7 +399,7 @@ function build_image {
     esac
 }
 
-OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@")
+OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -477,6 +481,10 @@ while true; do
             TAG_LATEST=yes
             shift
             ;;
+        --attempts)
+            MAX_ATTEMPTS=$2
+            shift 2
+            ;;
         -h | --help )
             usage
             exit 1
@@ -555,6 +563,9 @@ fi
 docker images --format '{{.Repository}}:{{.Tag}}' ${BASE} | grep -q "^${BASE}$"
 BASE_IMAGE_PRESENT=$?
 
+# Pull the image anyway, to ensure it's up to date
+docker pull ${BASE}
+
 # Download loci, if needed.
 get_loci
 if [ $? -ne 0 ]; then
@@ -573,7 +584,7 @@ for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_
     done
 done
 
-if [ "${CLEAN}" = "yes" ]; then
+if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then
     # Delete the images
     echo "Deleting images"
     docker image rm ${RESULTS_BUILT[@]} ${RESULTS_PUSHED[@]}

build-tools/build-wheels/build-base-wheels.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2018 Wind River Systems, Inc.
+# Copyright (c) 2018-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -8,6 +8,10 @@
 # for a set of upstream python modules.
 #
 
+MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
+
+source ${MY_SCRIPT_DIR}/utils.sh
+
 # Required env vars
 if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
     echo "Environment not setup for builds" >&2
@@ -21,6 +25,7 @@ OS=centos
 OS_VERSION=7.5.1804
 BUILD_STREAM=stable
 PROXY=""
+declare -i MAX_ATTEMPTS=1
 
 function usage {
     cat >&2 <<EOF
@@ -34,11 +39,12 @@ Options:
     --keep-container: Skip deletion of container used for the build
     --proxy:          Set proxy <URL>:<PORT>
     --stream:         Build stream, stable or dev (default: stable)
+    --attempts:       Max attempts, in case of failure (default: 1)
 
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy:,attempts: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -81,6 +87,10 @@ while true; do
             PROXY=$2
             shift 2
             ;;
+        --attempts)
+            MAX_ATTEMPTS=$2
+            shift 2
+            ;;
         -h | --help )
             usage
             exit 1
@@ -210,7 +220,7 @@ BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME})
 BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH})
 
 # Build image
-docker build "${BUILD_ARGS[@]}"
+with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
 if [ $? -ne 0 ]; then
     echo "Failed to create build image in docker" >&2
     exit 1
@@ -230,7 +240,7 @@ fi
 RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh)
 
 # Run container to build wheels
-docker run ${RUN_ARGS[@]}
+with_retries ${MAX_ATTEMPTS} docker run ${RUN_ARGS[@]}
 
 if [ "${KEEP_IMAGE}" = "no" ]; then
     # Delete the builder image

build-tools/build-wheels/build-wheel-tarball.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2018 Wind River Systems, Inc.
+# Copyright (c) 2018-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -9,6 +9,8 @@
 
 MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
 
+source ${MY_SCRIPT_DIR}/utils.sh
+
 # Required env vars
 if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
     echo "Environment not setup for builds" >&2
@@ -25,6 +27,7 @@ PUSH=no
 PROXY=""
 CLEAN=no
 DOCKER_USER=${USER}
+declare -i MAX_ATTEMPTS=1
 
 # List of top-level services for images, which should not be listed in upper-constraints.txt
 SKIP_CONSTRAINTS=(
@@ -56,11 +59,12 @@ Options:
     --proxy:      Set proxy <URL>:<PORT>
     --user:       Docker repo userid
     --version:    Version for pushed image (if used with --push)
+    --attempts:   Max attempts, in case of failure (default: 1)
 
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version:,attempts: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -111,6 +115,10 @@ while true; do
             VERSION=$2
             shift 2
             ;;
+        --attempts)
+            MAX_ATTEMPTS=$2
+            shift 2
+            ;;
         -h | --help )
             usage
             exit 1
@@ -143,7 +151,7 @@ if [ ! -z "$PROXY" ]; then
     BUILD_BASE_WL_ARGS+=(--proxy ${PROXY})
 fi
 
-${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]}
+${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS}
 if [ $? -ne 0 ]; then
     echo "Failure running build-base-wheels.sh" >&2
     exit 1
@@ -177,13 +185,13 @@ else
     OPENSTACK_BRANCH=stable/${CURRENT_STABLE_OPENSTACK}
 fi
 
-wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
+with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
 if [ $? -ne 0 ]; then
     echo "Failed to download global-requirements.txt" >&2
     exit 1
 fi
 
-wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
+with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
 if [ $? -ne 0 ]; then
     echo "Failed to download upper-constraints.txt" >&2
     exit 1

build-tools/build-wheels/docker/docker-build-wheel.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2018 Wind River Systems, Inc.
+# Copyright (c) 2018-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -11,6 +11,7 @@
 CFGFILE=/wheels.cfg
 OUTPUTDIR=/wheels
 FAILED_LOG=$OUTPUTDIR/failed.lst
+declare -i MAX_ATTEMPTS=5
 
 #
 # Function to log the start of a build
@@ -102,6 +103,41 @@ function fix_setup {
 
 }
 
+#
+# Function to call a command, with support for retries
+#
+function with_retries {
+    local max_attempts=$1
+    local cmd=$2
+
+    # Pop the first two arguments off the list,
+    # so we can pass additional args to the command safely
+    shift 2
+
+    local -i attempt=0
+
+    while :; do
+        let -i attempt++
+
+        echo "Running: ${cmd} $@"
+        ${cmd} "$@"
+        if [ $? -eq 0 ]; then
+            return 0
+        fi
+
+        echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
+        if [ ${attempt} -lt ${max_attempts} ]; then
+            local delay=5
+            echo "Waiting ${delay} seconds before retrying..."
+            sleep ${delay}
+            continue
+        else
+            echo "Max command attempts reached. Aborting..."
+            return 1
+        fi
+    done
+}
+
 #
 # Function to use git to clone the module source and build a wheel.
 #
@@ -176,9 +212,8 @@ function from_tar {
             taropts="xf"
         fi
 
-        wget $wgetsrc
+        with_retries ${MAX_ATTEMPTS} wget $wgetsrc
         if [ $? -ne 0 ]; then
-            echo "Failure running: wget $wgetsrc"
             echo $wheelname >> $FAILED_LOG
             continue
         fi
@@ -228,9 +263,8 @@ function from_zip {
             continue
         fi
 
-        wget $wgetsrc
+        with_retries ${MAX_ATTEMPTS} wget $wgetsrc
         if [ $? -ne 0 ]; then
-            echo "Failure running: wget $wgetsrc"
             echo $wheelname >> $FAILED_LOG
             continue
         fi
@@ -280,9 +314,8 @@ function from_pypi {
             continue
         fi
 
-        wget $wgetsrc
+        with_retries ${MAX_ATTEMPTS} wget $wgetsrc
         if [ $? -ne 0 ]; then
-            echo "Failure running: wget $wgetsrc"
             echo $wheelname >> $FAILED_LOG
             continue
         fi

build-tools/build-wheels/utils.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Image and wheel build utility functions
+#
+
+#
+# Function to call a command, with support for retries
+#
+function with_retries {
+    local max_attempts=$1
+    local cmd=$2
+
+    # Pop the first two arguments off the list,
+    # so we can pass additional args to the command safely
+    shift 2
+
+    local -i attempt=0
+
+    while :; do
+        let -i attempt++
+
+        echo "Running: ${cmd} $@"
+        ${cmd} "$@"
+        if [ $? -eq 0 ]; then
+            return 0
+        fi
+
+        echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
+        if [ ${attempt} -lt ${max_attempts} ]; then
+            local delay=5
+            echo "Waiting ${delay} seconds before retrying..."
+            sleep ${delay}
+            continue
+        else
+            echo "Max command attempts reached. Aborting..."
+            return 1
+        fi
+    done
+}
+