starlingx/root
Code Issues Proposed changes

Retries for certain wheel and image build commands

Browse Source 
The wheel and image build tools hit intermittent issues such as:
* Network or server issues that cause file download failures
* Repo access failures due to repo updates or other issues

To avoid formal build failures on such issues, this update adds
retries around specific commands, like wget and docker build.

Change-Id: Ifafdc6201872f43b2dd77efd4dcb033456477c2e
Closes-Bug: 1823986
Signed-off-by: Don Penney <don.penney@windriver.com>
This commit is contained in:
Don Penney 2019-04-09 16:17:22 -04:00
parent 02fa17a6ba
commit 492b5b218d
6 changed files with 143 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
build-tools/build-docker-images/build-stx-base.sh
View File

@ -1,14 +1,16 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This utility builds the StarlingX wheel tarball
# This utility builds the StarlingX base image
#
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
@ -31,6 +33,7 @@ CLEAN=no
TAG_LATEST=no
LATEST_TAG=latest
HOST=${HOSTNAME}
declare -i MAX_ATTEMPTS=1
function usage {
cat >&2 <<EOF
@ -52,11 +55,12 @@ Options:
--registry: Docker registry
--clean: Remove image(s) from local registry
--hostname: build repo host
--attempts: Max attempts, in case of failure (default: 1)
EOF
}
OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@")
OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname:,attempts: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -132,6 +136,10 @@ while true; do
HOST=$2
shift 2
;;
--attempts)
MAX_ATTEMPTS=$2
shift 2
;;
-h | --help )
usage
exit 1
@ -223,6 +231,9 @@ done
docker images --format '{{.Repository}}:{{.Tag}}' ${OS}:${OS_VERSION} | grep -q "^${OS}:${OS_VERSION}$"
BASE_IMAGE_PRESENT=$?
# Pull the image anyway, to ensure it is up to date
docker pull ${OS}:${OS_VERSION}
# Build the image
IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION}
IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG}
@ -238,7 +249,7 @@ fi
BUILD_ARGS+=(--tag ${IMAGE_NAME} ${BUILDDIR})
# Build base image
docker build "${BUILD_ARGS[@]}"
with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
if [ $? -ne 0 ]; then
echo "Failed running docker build command" >&2
exit 1

23
build-tools/build-docker-images/build-stx-images.sh
View File

@ -1,14 +1,16 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This utility builds the StarlingX wheel tarball
# This utility builds the StarlingX container images
#
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
@ -35,6 +37,7 @@ TAG_LIST_FILE=
TAG_LIST_LATEST_FILE=
declare -a ONLY
declare -a SKIP
declare -i MAX_ATTEMPTS=1
function usage {
cat >&2 <<EOF
@ -61,6 +64,7 @@ Options:
--skip <image> : Skip building the specified image(s). Multiple images
can be specified with a comma-separated list, or with
multiple --skip arguments.
--attempts: Max attempts, in case of failure (default: 1)
EOF
@ -276,7 +280,7 @@ function build_image_loci {
local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}"
docker build ${WORKDIR}/loci --no-cache \
with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci --no-cache \
"${BUILD_ARGS[@]}" \
--tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
if [ ${PIPESTATUS[0]} -ne 0 ]; then
@ -361,7 +365,7 @@ function build_image_docker {
BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY)
fi
BASE_BUILD_ARGS+=(--tag ${build_image_name})
docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
if [ ${PIPESTATUS[0]} -ne 0 ]; then
echo "Failed to build ${LABEL}... Aborting"
@ -395,7 +399,7 @@ function build_image {
esac
}
OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@")
OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -477,6 +481,10 @@ while true; do
TAG_LATEST=yes
shift
;;
--attempts)
MAX_ATTEMPTS=$2
shift 2
;;
-h | --help )
usage
exit 1
@ -555,6 +563,9 @@ fi
docker images --format '{{.Repository}}:{{.Tag}}' ${BASE} | grep -q "^${BASE}$"
BASE_IMAGE_PRESENT=$?
# Pull the image anyway, to ensure it's up to date
docker pull ${BASE}
# Download loci, if needed.
get_loci
if [ $? -ne 0 ]; then
@ -573,7 +584,7 @@ for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_
done
done
if [ "${CLEAN}" = "yes" ]; then
if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then
# Delete the images
echo "Deleting images"
docker image rm ${RESULTS_BUILT[@]} ${RESULTS_PUSHED[@]}

18
build-tools/build-wheels/build-base-wheels.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -8,6 +8,10 @@
# for a set of upstream python modules.
#
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
@ -21,6 +25,7 @@ OS=centos
OS_VERSION=7.5.1804
BUILD_STREAM=stable
PROXY=""
declare -i MAX_ATTEMPTS=1
function usage {
cat >&2 <<EOF
@ -34,11 +39,12 @@ Options:
--keep-container: Skip deletion of container used for the build
--proxy: Set proxy <URL>:<PORT>
--stream: Build stream, stable or dev (default: stable)
--attempts: Max attempts, in case of failure (default: 1)
EOF
}
OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy: -- "$@")
OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy:,attempts: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -81,6 +87,10 @@ while true; do
PROXY=$2
shift 2
;;
--attempts)
MAX_ATTEMPTS=$2
shift 2
;;
-h | --help )
usage
exit 1
@ -210,7 +220,7 @@ BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME})
BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH})
# Build image
docker build "${BUILD_ARGS[@]}"
with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
if [ $? -ne 0 ]; then
echo "Failed to create build image in docker" >&2
exit 1
@ -230,7 +240,7 @@ fi
RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh)
# Run container to build wheels
docker run ${RUN_ARGS[@]}
with_retries ${MAX_ATTEMPTS} docker run ${RUN_ARGS[@]}
if [ "${KEEP_IMAGE}" = "no" ]; then
# Delete the builder image

18
build-tools/build-wheels/build-wheel-tarball.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -9,6 +9,8 @@
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
@ -25,6 +27,7 @@ PUSH=no
PROXY=""
CLEAN=no
DOCKER_USER=${USER}
declare -i MAX_ATTEMPTS=1
# List of top-level services for images, which should not be listed in upper-constraints.txt
SKIP_CONSTRAINTS=(
@ -56,11 +59,12 @@ Options:
--proxy: Set proxy <URL>:<PORT>
--user: Docker repo userid
--version: Version for pushed image (if used with --push)
--attempts: Max attempts, in case of failure (default: 1)
EOF
}
OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version: -- "$@")
OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version:,attempts: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -111,6 +115,10 @@ while true; do
VERSION=$2
shift 2
;;
--attempts)
MAX_ATTEMPTS=$2
shift 2
;;
-h | --help )
usage
exit 1
@ -143,7 +151,7 @@ if [ ! -z "$PROXY" ]; then
BUILD_BASE_WL_ARGS+=(--proxy ${PROXY})
fi
${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]}
${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS}
if [ $? -ne 0 ]; then
echo "Failure running build-base-wheels.sh" >&2
exit 1
@ -177,13 +185,13 @@ else
OPENSTACK_BRANCH=stable/${CURRENT_STABLE_OPENSTACK}
fi
wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
if [ $? -ne 0 ]; then
echo "Failed to download global-requirements.txt" >&2
exit 1
fi
wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
if [ $? -ne 0 ]; then
echo "Failed to download upper-constraints.txt" >&2
exit 1

47
build-tools/build-wheels/docker/docker-build-wheel.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -11,6 +11,7 @@
CFGFILE=/wheels.cfg
OUTPUTDIR=/wheels
FAILED_LOG=$OUTPUTDIR/failed.lst
declare -i MAX_ATTEMPTS=5
#
# Function to log the start of a build
@ -102,6 +103,41 @@ function fix_setup {
}
#
# Function to call a command, with support for retries
#
function with_retries {
local max_attempts=$1
local cmd=$2
# Pop the first two arguments off the list,
# so we can pass additional args to the command safely
shift 2
local -i attempt=0
while :; do
let -i attempt++
echo "Running: ${cmd} $@"
${cmd} "$@"
if [ $? -eq 0 ]; then
return 0
fi
echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
if [ ${attempt} -lt ${max_attempts} ]; then
local delay=5
echo "Waiting ${delay} seconds before retrying..."
sleep ${delay}
continue
else
echo "Max command attempts reached. Aborting..."
return 1
fi
done
}
#
# Function to use git to clone the module source and build a wheel.
#
@ -176,9 +212,8 @@ function from_tar {
taropts="xf"
fi
wget $wgetsrc
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
if [ $? -ne 0 ]; then
echo "Failure running: wget $wgetsrc"
echo $wheelname >> $FAILED_LOG
continue
fi
@ -228,9 +263,8 @@ function from_zip {
continue
fi
wget $wgetsrc
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
if [ $? -ne 0 ]; then
echo "Failure running: wget $wgetsrc"
echo $wheelname >> $FAILED_LOG
continue
fi
@ -280,9 +314,8 @@ function from_pypi {
continue
fi
wget $wgetsrc
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
if [ $? -ne 0 ]; then
echo "Failure running: wget $wgetsrc"
echo $wheelname >> $FAILED_LOG
continue
fi

44
build-tools/build-wheels/utils.sh Normal file
View File

@ -0,0 +1,44 @@
#!/bin/bash
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# Image and wheel build utility functions
#
#
# Function to call a command, with support for retries
#
function with_retries {
local max_attempts=$1
local cmd=$2
# Pop the first two arguments off the list,
# so we can pass additional args to the command safely
shift 2
local -i attempt=0
while :; do
let -i attempt++
echo "Running: ${cmd} $@"
${cmd} "$@"
if [ $? -eq 0 ]; then
return 0
fi
echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
if [ ${attempt} -lt ${max_attempts} ]; then
local delay=5
echo "Waiting ${delay} seconds before retrying..."
sleep ${delay}
continue
else
echo "Max command attempts reached. Aborting..."
return 1
fi
done
}