Retries for certain wheel and image build commands
The wheel and image build tools hit intermittent issues such as: * Network or server issues that cause file download failures * Repo access failures due to repo updates or other issues To avoid formal build failures on such issues, this update adds retries around specific commands, like wget and docker build. Change-Id: Ifafdc6201872f43b2dd77efd4dcb033456477c2e Closes-Bug: 1823986 Signed-off-by: Don Penney <don.penney@windriver.com>
This commit is contained in:
Don Penney
parent
02fa17a6ba
commit
492b5b218d
|
|
@ -1,14 +1,16 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2018 Wind River Systems, Inc.
|
||||
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# This utility builds the StarlingX wheel tarball
|
||||
# This utility builds the StarlingX base image
|
||||
#
|
||||
|
||||
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
|
||||
|
||||
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
|
||||
|
||||
# Required env vars
|
||||
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
|
||||
echo "Environment not setup for builds" >&2
|
||||
|
|
@ -31,6 +33,7 @@ CLEAN=no
|
|||
TAG_LATEST=no
|
||||
LATEST_TAG=latest
|
||||
HOST=${HOSTNAME}
|
||||
declare -i MAX_ATTEMPTS=1
|
||||
|
||||
function usage {
|
||||
cat >&2 <<EOF
|
||||
|
|
@ -52,11 +55,12 @@ Options:
|
|||
--registry: Docker registry
|
||||
--clean: Remove image(s) from local registry
|
||||
--hostname: build repo host
|
||||
--attempts: Max attempts, in case of failure (default: 1)
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@")
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname:,attempts: -- "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -132,6 +136,10 @@ while true; do
|
|||
HOST=$2
|
||||
shift 2
|
||||
;;
|
||||
--attempts)
|
||||
MAX_ATTEMPTS=$2
|
||||
shift 2
|
||||
;;
|
||||
-h | --help )
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -223,6 +231,9 @@ done
|
|||
docker images --format '{{.Repository}}:{{.Tag}}' ${OS}:${OS_VERSION} | grep -q "^${OS}:${OS_VERSION}$"
|
||||
BASE_IMAGE_PRESENT=$?
|
||||
|
||||
# Pull the image anyway, to ensure it is up to date
|
||||
docker pull ${OS}:${OS_VERSION}
|
||||
|
||||
# Build the image
|
||||
IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION}
|
||||
IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG}
|
||||
|
|
@ -238,7 +249,7 @@ fi
|
|||
BUILD_ARGS+=(--tag ${IMAGE_NAME} ${BUILDDIR})
|
||||
|
||||
# Build base image
|
||||
docker build "${BUILD_ARGS[@]}"
|
||||
with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed running docker build command" >&2
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2018 Wind River Systems, Inc.
|
||||
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# This utility builds the StarlingX wheel tarball
|
||||
# This utility builds the StarlingX container images
|
||||
#
|
||||
|
||||
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
|
||||
|
||||
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
|
||||
|
||||
# Required env vars
|
||||
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
|
||||
echo "Environment not setup for builds" >&2
|
||||
|
|
@ -35,6 +37,7 @@ TAG_LIST_FILE=
|
|||
TAG_LIST_LATEST_FILE=
|
||||
declare -a ONLY
|
||||
declare -a SKIP
|
||||
declare -i MAX_ATTEMPTS=1
|
||||
|
||||
function usage {
|
||||
cat >&2 <<EOF
|
||||
|
|
@ -61,6 +64,7 @@ Options:
|
|||
--skip <image> : Skip building the specified image(s). Multiple images
|
||||
can be specified with a comma-separated list, or with
|
||||
multiple --skip arguments.
|
||||
--attempts: Max attempts, in case of failure (default: 1)
|
||||
|
||||
|
||||
EOF
|
||||
|
|
@ -276,7 +280,7 @@ function build_image_loci {
|
|||
|
||||
local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}"
|
||||
|
||||
docker build ${WORKDIR}/loci --no-cache \
|
||||
with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci --no-cache \
|
||||
"${BUILD_ARGS[@]}" \
|
||||
--tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
|
||||
if [ ${PIPESTATUS[0]} -ne 0 ]; then
|
||||
|
|
@ -361,7 +365,7 @@ function build_image_docker {
|
|||
BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY)
|
||||
fi
|
||||
BASE_BUILD_ARGS+=(--tag ${build_image_name})
|
||||
docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
|
||||
with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
|
||||
|
||||
if [ ${PIPESTATUS[0]} -ne 0 ]; then
|
||||
echo "Failed to build ${LABEL}... Aborting"
|
||||
|
|
@ -395,7 +399,7 @@ function build_image {
|
|||
esac
|
||||
}
|
||||
|
||||
OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@")
|
||||
OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -477,6 +481,10 @@ while true; do
|
|||
TAG_LATEST=yes
|
||||
shift
|
||||
;;
|
||||
--attempts)
|
||||
MAX_ATTEMPTS=$2
|
||||
shift 2
|
||||
;;
|
||||
-h | --help )
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -555,6 +563,9 @@ fi
|
|||
docker images --format '{{.Repository}}:{{.Tag}}' ${BASE} | grep -q "^${BASE}$"
|
||||
BASE_IMAGE_PRESENT=$?
|
||||
|
||||
# Pull the image anyway, to ensure it's up to date
|
||||
docker pull ${BASE}
|
||||
|
||||
# Download loci, if needed.
|
||||
get_loci
|
||||
if [ $? -ne 0 ]; then
|
||||
|
|
@ -573,7 +584,7 @@ for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_
|
|||
done
|
||||
done
|
||||
|
||||
if [ "${CLEAN}" = "yes" ]; then
|
||||
if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then
|
||||
# Delete the images
|
||||
echo "Deleting images"
|
||||
docker image rm ${RESULTS_BUILT[@]} ${RESULTS_PUSHED[@]}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2018 Wind River Systems, Inc.
|
||||
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
|
@ -8,6 +8,10 @@
|
|||
# for a set of upstream python modules.
|
||||
#
|
||||
|
||||
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
|
||||
|
||||
source ${MY_SCRIPT_DIR}/utils.sh
|
||||
|
||||
# Required env vars
|
||||
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
|
||||
echo "Environment not setup for builds" >&2
|
||||
|
|
@ -21,6 +25,7 @@ OS=centos
|
|||
OS_VERSION=7.5.1804
|
||||
BUILD_STREAM=stable
|
||||
PROXY=""
|
||||
declare -i MAX_ATTEMPTS=1
|
||||
|
||||
function usage {
|
||||
cat >&2 <<EOF
|
||||
|
|
@ -34,11 +39,12 @@ Options:
|
|||
--keep-container: Skip deletion of container used for the build
|
||||
--proxy: Set proxy <URL>:<PORT>
|
||||
--stream: Build stream, stable or dev (default: stable)
|
||||
--attempts: Max attempts, in case of failure (default: 1)
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy: -- "$@")
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy:,attempts: -- "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -81,6 +87,10 @@ while true; do
|
|||
PROXY=$2
|
||||
shift 2
|
||||
;;
|
||||
--attempts)
|
||||
MAX_ATTEMPTS=$2
|
||||
shift 2
|
||||
;;
|
||||
-h | --help )
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -210,7 +220,7 @@ BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME})
|
|||
BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH})
|
||||
|
||||
# Build image
|
||||
docker build "${BUILD_ARGS[@]}"
|
||||
with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to create build image in docker" >&2
|
||||
exit 1
|
||||
|
|
@ -230,7 +240,7 @@ fi
|
|||
RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh)
|
||||
|
||||
# Run container to build wheels
|
||||
docker run ${RUN_ARGS[@]}
|
||||
with_retries ${MAX_ATTEMPTS} docker run ${RUN_ARGS[@]}
|
||||
|
||||
if [ "${KEEP_IMAGE}" = "no" ]; then
|
||||
# Delete the builder image
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2018 Wind River Systems, Inc.
|
||||
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
|
@ -9,6 +9,8 @@
|
|||
|
||||
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
|
||||
|
||||
source ${MY_SCRIPT_DIR}/utils.sh
|
||||
|
||||
# Required env vars
|
||||
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
|
||||
echo "Environment not setup for builds" >&2
|
||||
|
|
@ -25,6 +27,7 @@ PUSH=no
|
|||
PROXY=""
|
||||
CLEAN=no
|
||||
DOCKER_USER=${USER}
|
||||
declare -i MAX_ATTEMPTS=1
|
||||
|
||||
# List of top-level services for images, which should not be listed in upper-constraints.txt
|
||||
SKIP_CONSTRAINTS=(
|
||||
|
|
@ -56,11 +59,12 @@ Options:
|
|||
--proxy: Set proxy <URL>:<PORT>
|
||||
--user: Docker repo userid
|
||||
--version: Version for pushed image (if used with --push)
|
||||
--attempts: Max attempts, in case of failure (default: 1)
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version: -- "$@")
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version:,attempts: -- "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -111,6 +115,10 @@ while true; do
|
|||
VERSION=$2
|
||||
shift 2
|
||||
;;
|
||||
--attempts)
|
||||
MAX_ATTEMPTS=$2
|
||||
shift 2
|
||||
;;
|
||||
-h | --help )
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -143,7 +151,7 @@ if [ ! -z "$PROXY" ]; then
|
|||
BUILD_BASE_WL_ARGS+=(--proxy ${PROXY})
|
||||
fi
|
||||
|
||||
${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]}
|
||||
${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failure running build-base-wheels.sh" >&2
|
||||
exit 1
|
||||
|
|
@ -177,13 +185,13 @@ else
|
|||
OPENSTACK_BRANCH=stable/${CURRENT_STABLE_OPENSTACK}
|
||||
fi
|
||||
|
||||
wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
|
||||
with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to download global-requirements.txt" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
|
||||
with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to download upper-constraints.txt" >&2
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2018 Wind River Systems, Inc.
|
||||
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
|
@ -11,6 +11,7 @@
|
|||
CFGFILE=/wheels.cfg
|
||||
OUTPUTDIR=/wheels
|
||||
FAILED_LOG=$OUTPUTDIR/failed.lst
|
||||
declare -i MAX_ATTEMPTS=5
|
||||
|
||||
#
|
||||
# Function to log the start of a build
|
||||
|
|
@ -102,6 +103,41 @@ function fix_setup {
|
|||
|
||||
}
|
||||
|
||||
#
|
||||
# Function to call a command, with support for retries
|
||||
#
|
||||
function with_retries {
|
||||
local max_attempts=$1
|
||||
local cmd=$2
|
||||
|
||||
# Pop the first two arguments off the list,
|
||||
# so we can pass additional args to the command safely
|
||||
shift 2
|
||||
|
||||
local -i attempt=0
|
||||
|
||||
while :; do
|
||||
let -i attempt++
|
||||
|
||||
echo "Running: ${cmd} $@"
|
||||
${cmd} "$@"
|
||||
if [ $? -eq 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
|
||||
if [ ${attempt} -lt ${max_attempts} ]; then
|
||||
local delay=5
|
||||
echo "Waiting ${delay} seconds before retrying..."
|
||||
sleep ${delay}
|
||||
continue
|
||||
else
|
||||
echo "Max command attempts reached. Aborting..."
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
#
|
||||
# Function to use git to clone the module source and build a wheel.
|
||||
#
|
||||
|
|
@ -176,9 +212,8 @@ function from_tar {
|
|||
taropts="xf"
|
||||
fi
|
||||
|
||||
wget $wgetsrc
|
||||
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failure running: wget $wgetsrc"
|
||||
echo $wheelname >> $FAILED_LOG
|
||||
continue
|
||||
fi
|
||||
|
|
@ -228,9 +263,8 @@ function from_zip {
|
|||
continue
|
||||
fi
|
||||
|
||||
wget $wgetsrc
|
||||
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failure running: wget $wgetsrc"
|
||||
echo $wheelname >> $FAILED_LOG
|
||||
continue
|
||||
fi
|
||||
|
|
@ -280,9 +314,8 @@ function from_pypi {
|
|||
continue
|
||||
fi
|
||||
|
||||
wget $wgetsrc
|
||||
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failure running: wget $wgetsrc"
|
||||
echo $wheelname >> $FAILED_LOG
|
||||
continue
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Image and wheel build utility functions
|
||||
#
|
||||
|
||||
#
|
||||
# Function to call a command, with support for retries
|
||||
#
|
||||
function with_retries {
|
||||
local max_attempts=$1
|
||||
local cmd=$2
|
||||
|
||||
# Pop the first two arguments off the list,
|
||||
# so we can pass additional args to the command safely
|
||||
shift 2
|
||||
|
||||
local -i attempt=0
|
||||
|
||||
while :; do
|
||||
let -i attempt++
|
||||
|
||||
echo "Running: ${cmd} $@"
|
||||
${cmd} "$@"
|
||||
if [ $? -eq 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
|
||||
if [ ${attempt} -lt ${max_attempts} ]; then
|
||||
local delay=5
|
||||
echo "Waiting ${delay} seconds before retrying..."
|
||||
sleep ${delay}
|
||||
continue
|
||||
else
|
||||
echo "Max command attempts reached. Aborting..."
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue