Merge "build-docker-images: add retry delay"

This commit is contained in:
Zuul 2023-05-30 14:23:52 +00:00 committed by Gerrit Code Review
commit c0a4a59ab3
6 changed files with 88 additions and 93 deletions

View File

@ -9,7 +9,7 @@
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
source ${MY_SCRIPT_DIR}/../utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
@ -38,6 +38,7 @@ LATEST_TAG=latest
HOST=${HOSTNAME}
USE_DOCKER_CACHE=no
declare -i MAX_ATTEMPTS=1
declare -i RETRY_DELAY=30
function usage {
cat >&2 <<EOF
@ -63,7 +64,10 @@ Options:
--registry: Docker registry
--clean: Remove image(s) from local registry
--hostname: build repo host
--attempts: Max attempts, in case of failure (default: 1)
--attempts <count>
Max attempts, in case of failure (default: 1)
--retry-delay <seconds>
Sleep this many seconds between retries (default: 30)
--config-file:Specify a path to a config file which will specify additional arguments to be passed into the command
--cache: Allow docker to use cached filesystem layers when building
@ -118,7 +122,7 @@ function get_args_from_file {
done <"$1"
}
OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,cache,hostname:,attempts:,config-file: -- "$@")
OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,cache,hostname:,attempts:,retry-delay:,config-file: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -202,6 +206,10 @@ while true; do
MAX_ATTEMPTS=$2
shift 2
;;
--retry-delay)
RETRY_DELAY=$2
shift 2
;;
--config-file)
CONFIG_FILE=$2
shift 2
@ -404,7 +412,7 @@ fi
BUILD_ARGS+=(--tag ${IMAGE_NAME} ${BUILDDIR})
# Build base image
with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
if [ $? -ne 0 ]; then
echo "Failed running docker build command" >&2
exit 1
@ -413,7 +421,7 @@ fi
if [ "${PUSH}" = "yes" ]; then
# Push the image
echo "Pushing image: ${IMAGE_NAME}"
docker push ${IMAGE_NAME}
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker push ${IMAGE_NAME}
if [ $? -ne 0 ]; then
echo "Failed running docker push command" >&2
exit 1
@ -422,7 +430,7 @@ if [ "${PUSH}" = "yes" ]; then
if [ "$TAG_LATEST" = "yes" ]; then
docker tag ${IMAGE_NAME} ${IMAGE_NAME_LATEST}
echo "Pushing image: ${IMAGE_NAME_LATEST}"
docker push ${IMAGE_NAME_LATEST}
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker push ${IMAGE_NAME_LATEST}
if [ $? -ne 0 ]; then
echo "Failed running docker push command on latest" >&2
exit 1

View File

@ -9,8 +9,6 @@
MY_SCRIPT_DIR=$(dirname $(readlink -fv $0))
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
@ -45,6 +43,7 @@ DEFAULT_SPICE_REPO="https://gitlab.freedesktop.org/spice/spice-html5"
declare -a ONLY
declare -a SKIP
declare -i MAX_ATTEMPTS=1
declare -i RETRY_DELAY=30
declare -a RESULTS_BUILT
declare -a RESULTS_PUSHED
@ -89,7 +88,10 @@ Options:
--skip <image> : Skip building the specified image(s). Multiple images
can be specified with a comma-separated list, or with
multiple --skip arguments.
--attempts: Max attempts, in case of failure (default: 1)
--attempts <count>
How many times to try a failed build command (default: 1)
--retry-delay <seconds>
Sleep this many seconds between retries (default: 30)
--cache: Allow docker to use filesystem cache when building
CAUTION: this option may ignore locally-generated
@ -180,11 +182,13 @@ function get_git {
if [ ! -d ${WORKDIR}/${git_name} ]; then
cd ${WORKDIR}
git clone --recursive ${git_repo}
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} $SHELL -c "rm -rf ${git_name}.clone_tmp && git clone --recursive ${git_repo} ${git_name}.clone_tmp"
if [ $? -ne 0 ]; then
rm -rf ${git_name}.clone_tmp
echo "Failed to clone ${git_repo}. Aborting..." >&2
return 1
fi
mv ${git_name}.clone_tmp ${git_name}
cd $git_name
git checkout ${git_ref}
@ -206,7 +210,7 @@ function get_git {
else
cd ${WORKDIR}/${git_name}
git fetch
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} git fetch
if [ $? -ne 0 ]; then
echo "Failed to fetch '${git_name}'. Aborting..." >&2
return 1
@ -347,7 +351,7 @@ function post_build {
local push_tag="${DOCKER_REGISTRY}${DOCKER_USER}/${LABEL}:${IMAGE_TAG_VERSIONED}"
docker tag ${build_image_name} ${push_tag}
with_retries ${MAX_ATTEMPTS} docker push ${push_tag}
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker push ${push_tag}
if [[ $? -ne 0 ]] ; then
echo "Failed to push ${push_tag} ... Aborting"
RESULTS_PUSH_FAILED+=(${LABEL})
@ -361,7 +365,7 @@ function post_build {
if [ "$TAG_LATEST" = "yes" ]; then
local latest_tag="${DOCKER_REGISTRY}${DOCKER_USER}/${LABEL}:${IMAGE_TAG_LATEST}"
docker tag ${push_tag} ${latest_tag}
with_retries ${MAX_ATTEMPTS} docker push ${latest_tag}
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker push ${latest_tag}
if [[ $? -ne 0 ]] ; then
echo "Failed to push ${latest_tag} ... Aborting"
RESULTS_PUSH_FAILED+=(${LABEL})
@ -457,7 +461,8 @@ function build_image_loci {
echo "Creating bare clone of ${PROJECT_REPO} for ${LABEL} build..."
if [ -n "${PROJECT_REF}" ]; then
echo "PROJECT_REF specified is ${PROJECT_REF}..."
git clone --no-local --bare ${PROJECT_REPO} ${CLONE_DIR} \
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} $SHELL -c "rm -rf ${CLONE_DIR}.clone_tmp && git clone --no-local --bare ${PROJECT_REPO} ${CLONE_DIR}.clone_tmp" \
&& mv ${CLONE_DIR}.clone_tmp ${CLONE_DIR} \
&& cd ${PROJECT_REPO} \
&& git push --force ${CLONE_DIR} HEAD:refs/heads/${PROJECT_REF} \
&& mv ${CLONE_DIR}/hooks/post-update.sample ${CLONE_DIR}/hooks/post-update \
@ -466,7 +471,8 @@ function build_image_loci {
&& git update-server-info \
&& cd ${ORIGWD}
else
git clone --no-local --bare ${PROJECT_REPO} ${CLONE_DIR} \
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} $SHELL -c "rm -rf ${CLONE_DIR}.clone_tmp && git clone --no-local --bare ${PROJECT_REPO} ${CLONE_DIR}.clone_tmp" \
&& mv ${CLONE_DIR}.clone_tmp ${CLONE_DIR} \
&& cd ${PROJECT_REPO} \
&& mv ${CLONE_DIR}/hooks/post-update.sample ${CLONE_DIR}/hooks/post-update \
&& chmod a+x ${CLONE_DIR}/hooks/post-update \
@ -476,6 +482,7 @@ function build_image_loci {
fi
if [ $? -ne 0 ]; then
rm -rf ${CLONE_DIR}.clone_tmp
echo "Failed to clone ${PROJECT_REPO}... Aborting ${LABEL} build"
RESULTS_FAILED+=(${LABEL})
cd ${ORIGWD}
@ -571,7 +578,7 @@ function build_image_loci {
local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}"
with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci \
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci \
"${BUILD_ARGS[@]}" \
--tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS_LABEL}-${BUILD_STREAM}.log
if [ ${PIPESTATUS[0]} -ne 0 ]; then
@ -704,7 +711,7 @@ function build_image_docker {
fi
BASE_BUILD_ARGS+=(--tag ${build_image_name})
with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS_LABEL}-${BUILD_STREAM}.log
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS_LABEL}-${BUILD_STREAM}.log
if [ ${PIPESTATUS[0]} -ne 0 ]; then
echo "Failed to build ${LABEL}... Aborting"
@ -775,7 +782,7 @@ function build_image_script {
cp $(dirname ${image_build_file})/${SCRIPT} ${SCRIPT}
local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}"
with_retries ${MAX_ATTEMPTS} ${COMMAND} ${SCRIPT} ${ARGS} ${build_image_name} $HTTP_PROXY $HTTPS_PROXY $NO_PROXY 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS_LABEL}-${BUILD_STREAM}.log
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} ${COMMAND} ${SCRIPT} ${ARGS} ${build_image_name} $HTTP_PROXY $HTTPS_PROXY $NO_PROXY 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS_LABEL}-${BUILD_STREAM}.log
if [ ${PIPESTATUS[0]} -ne 0 ]; then
echo "Failed to build ${LABEL}... Aborting"
@ -817,7 +824,7 @@ function build_image {
esac
}
OPTS=$(getopt -o hN -l help,os:,os-label:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,wheels-py2:,only:,skip:,prefix:,latest,latest-prefix:,clean,cache,attempts:,no-pull-base -- "$@")
OPTS=$(getopt -o hN -l help,os:,os-label:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,wheels-py2:,only:,skip:,prefix:,latest,latest-prefix:,clean,cache,attempts:,retry-delay:,no-pull-base -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -923,6 +930,10 @@ while true; do
MAX_ATTEMPTS=$2
shift 2
;;
--retry-delay)
RETRY_DELAY=$2
shift 2
;;
-N|--no-pull-base)
PULL_BASE=no
shift
@ -1105,7 +1116,7 @@ BASE_IMAGE_PRESENT=$?
# Pull the image anyway, to ensure it's up to date
if [[ "$PULL_BASE" == "yes" ]] ; then
docker pull ${BASE}
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker pull ${BASE} || exit 1
fi
# Download loci, if needed.

View File

@ -9,7 +9,7 @@
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
source ${MY_SCRIPT_DIR}/../utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
@ -35,6 +35,7 @@ declare -a DIST_PACKAGES
declare -a MODULE_SRC
declare -a EXTRA_FILES
declare -i MAX_ATTEMPTS=1
declare -i RETRY_DELAY=30
function usage {
@ -67,7 +68,8 @@ Options:
--user: Docker repo userid
--registry: Docker registry
--clean: Remove image(s) from local registry
--attempts: Max attempts, in case of failure (default: 1)
--attempts <count>: Max attempts, in case of failure (default: 1)
--retry-delay <seconds>: Sleep between retries (default: 30)
--update-id: Update ID
@ -95,7 +97,7 @@ function copy_files_to_workdir {
for f in $*; do
if [[ ${f} =~ ^(http|https|git): ]]; then
pushd ${destdir}
with_retries ${MAX_ATTEMPTS} wget ${f}
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} wget ${f}
if [ $? -ne 0 ]; then
echo "Failed to download $f to ${destdir}" >&2
exit 1
@ -188,7 +190,7 @@ function read_params_from_file {
FILE_BASEDIR=$(dirname ${FILE})
}
OPTS=$(getopt -o h -l help,file:,from:,wheel:,module-src:,pkg:,customize:,extra:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,clean,attempts:,update-id: -- "$@")
OPTS=$(getopt -o h -l help,file:,from:,wheel:,module-src:,pkg:,customize:,extra:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,clean,attempts:,retry-delay:,update-id: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -268,6 +270,10 @@ while true; do
MAX_ATTEMPTS=$2
shift 2
;;
--retry-delay)
RETRY_DELAY=$2
shift 2
;;
--update-id)
UPDATE_ID=$2
shift 2

View File

@ -10,7 +10,7 @@
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/utils.sh
source ${MY_SCRIPT_DIR}/../utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
@ -30,6 +30,7 @@ NO_PROXY=""
USE_DOCKER_CACHE=no
: ${PYTHON3:=python3}
declare -i MAX_ATTEMPTS=1
declare -i RETRY_DELAY=30
function usage {
cat >&2 <<EOF
@ -45,7 +46,10 @@ Options:
--https_proxy: Set https proxy <URL>:<PORT>, urls splitted by ","
--no_proxy: Set bypass list for proxy <URL>, urls splitted by ","
--stream: Build stream, stable or dev (default: stable)
--attempts: Max attempts, in case of failure (default: 1)
--attempts <count>
Max attempts, in case of failure (default: 1)
--retry-delay <seconds>
Sleep this many seconds between retries (default: 30)
--cache: Allow docker to use filesystem cache when building
CAUTION: this option may ignore locally-generated
@ -55,7 +59,7 @@ Options:
EOF
}
OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,http_proxy:,https_proxy:,no_proxy:,attempts:,cache -- "$@")
OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,http_proxy:,https_proxy:,no_proxy:,attempts:,retry-delay:,cache -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -110,6 +114,10 @@ while true; do
MAX_ATTEMPTS=$2
shift 2
;;
--retry-delay)
RETRY_DELAY=$2
shift 2
;;
--cache)
USE_DOCKER_CACHE=yes
shift
@ -238,7 +246,7 @@ if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
docker images --format '{{.Repository}}:{{.Tag}}' ${MASTER_WHEELS_IMAGE} | grep -q "^${MASTER_WHEELS_IMAGE}$"
MASTER_WHEELS_PRESENT=$?
docker pull ${MASTER_WHEELS_IMAGE}
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker pull ${MASTER_WHEELS_IMAGE}
if [ $? -ne 0 ]; then
echo "Failed to pull ${MASTER_WHEELS_IMAGE}" >&2
exit 1
@ -378,7 +386,7 @@ BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME})
BUILD_ARGS+=(-f ${DOCKER_BUILD_PATH}/${OS}/Dockerfile ${DOCKER_BUILD_PATH})
# Build image
with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
if [ $? -ne 0 ]; then
echo "Failed to create build image in docker" >&2
exit 1
@ -407,13 +415,13 @@ rm -f ${BUILD_OUTPUT_PATH_PY2}/failed.lst
notice "building python3 wheels"
log_prefix "[python3] " \
with_retries ${MAX_ATTEMPTS} \
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} \
docker run ${RUN_ARGS[@]} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh
BUILD_STATUS=$?
notice "building python2 wheels"
log_prefix "[python2] " \
with_retries ${MAX_ATTEMPTS} \
with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} \
docker run ${RUN_ARGS[@]} -v ${BUILD_OUTPUT_PATH_PY2}:/wheels --env PYTHON=python2 ${BUILD_IMAGE_NAME} /docker-build-wheel.sh
BUILD_STATUS_PY2=$?

View File

@ -9,7 +9,7 @@
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/utils.sh
source ${MY_SCRIPT_DIR}/../utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
@ -30,6 +30,7 @@ CLEAN=no
KEEP_IMAGE=no
DOCKER_USER=${USER}
declare -i MAX_ATTEMPTS=1
declare -i RETRY_DELAY=30
PYTHON2=no
USE_DOCKER_CACHE=no
EXTRA_WHEELS_DIR=
@ -64,28 +65,29 @@ Usage:
$(basename $0)
Options:
--os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]})
--os-version: Specify OS version
--stream: Build stream, stable or dev (default: stable)
--push: Push to docker repo
--http_proxy: Set http proxy <URL>:<PORT>, urls splitted by ","
--os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]})
--os-version: Specify OS version
--stream: Build stream, stable or dev (default: stable)
--push: Push to docker repo
--http_proxy: Set http proxy <URL>:<PORT>, urls splitted by ","
--https_proxy: Set https proxy <URL>:<PORT>, urls splitted by ","
--no_proxy: Set bypass list for proxy <URL>, urls splitted by ","
--user: Docker repo userid
--version: Version for pushed image (if used with --push)
--attempts: Max attempts, in case of failure (default: 1)
--python2: Build a python2 tarball
--no_proxy: Set bypass list for proxy <URL>, urls splitted by ","
--user: Docker repo userid
--version: Version for pushed image (if used with --push)
--attempts: Max attempts, in case of failure (default: 1)
--retry-delay: Sleep this many seconds between retries (default: 30)
--python2: Build a python2 tarball
--extra-wheels-dir: Directory containing additional .whl files
--keep-image: Don't delete wheel builder image at the end
--keep-image: Don't delete wheel builder image at the end
--cache: Allow docker to use filesystem cache when building
CAUTION: this option may ignore locally-generated
packages and is meant for debugging the build
scripts.
--cache: Allow docker to use filesystem cache when building
CAUTION: this option may ignore locally-generated
packages and is meant for debugging the build
scripts.
EOF
}
OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts:,python2,extra-wheels-dir:,keep-image,cache -- "$@")
OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts:,retry-delay:,python2,extra-wheels-dir:,keep-image,cache -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -148,6 +150,10 @@ while true; do
MAX_ATTEMPTS=$2
shift 2
;;
--retry-delay)
RETRY_DELAY=$2
shift 2
;;
--python2)
PYTHON2=yes
shift
@ -238,7 +244,7 @@ if [[ "$USE_DOCKER_CACHE" == "yes" ]] ; then
BUILD_BASE_WL_ARGS+=(--cache)
fi
${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS}
${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts "${MAX_ATTEMPTS}" --retry-delay "${RETRY_DELAY}"
if [ $? -ne 0 ]; then
echo "Failure running build-base-wheels.sh" >&2
exit 1

View File

@ -1,44 +0,0 @@
#!/bin/bash
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# Image and wheel build utility functions
#
#
# Function to call a command, with support for retries
#
function with_retries {
local max_attempts=$1
local cmd=$2
# Pop the first two arguments off the list,
# so we can pass additional args to the command safely
shift 2
local -i attempt=0
while :; do
let attempt++
echo "Running: ${cmd} $@"
${cmd} "$@"
if [ $? -eq 0 ]; then
return 0
fi
echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
if [ ${attempt} -lt ${max_attempts} ]; then
local delay=5
echo "Waiting ${delay} seconds before retrying..."
sleep ${delay}
continue
else
echo "Max command attempts reached. Aborting..."
return 1
fi
done
}