Merge "Retries for certain wheel and image build commands"
This commit is contained in:
Zuul
Gerrit Code Review
commit
c88c281fa1
|
|
@ -1,14 +1,16 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2018 Wind River Systems, Inc.
|
||||
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# This utility builds the StarlingX wheel tarball
|
||||
# This utility builds the StarlingX base image
|
||||
#
|
||||
|
||||
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
|
||||
|
||||
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
|
||||
|
||||
# Required env vars
|
||||
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
|
||||
echo "Environment not setup for builds" >&2
|
||||
|
|
@ -31,6 +33,7 @@ CLEAN=no
|
|||
TAG_LATEST=no
|
||||
LATEST_TAG=latest
|
||||
HOST=${HOSTNAME}
|
||||
declare -i MAX_ATTEMPTS=1
|
||||
|
||||
function usage {
|
||||
cat >&2 <<EOF
|
||||
|
|
@ -52,11 +55,12 @@ Options:
|
|||
--registry: Docker registry
|
||||
--clean: Remove image(s) from local registry
|
||||
--hostname: build repo host
|
||||
--attempts: Max attempts, in case of failure (default: 1)
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@")
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname:,attempts: -- "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -132,6 +136,10 @@ while true; do
|
|||
HOST=$2
|
||||
shift 2
|
||||
;;
|
||||
--attempts)
|
||||
MAX_ATTEMPTS=$2
|
||||
shift 2
|
||||
;;
|
||||
-h | --help )
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -223,6 +231,9 @@ done
|
|||
docker images --format '{{.Repository}}:{{.Tag}}' ${OS}:${OS_VERSION} | grep -q "^${OS}:${OS_VERSION}$"
|
||||
BASE_IMAGE_PRESENT=$?
|
||||
|
||||
# Pull the image anyway, to ensure it is up to date
|
||||
docker pull ${OS}:${OS_VERSION}
|
||||
|
||||
# Build the image
|
||||
IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION}
|
||||
IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG}
|
||||
|
|
@ -238,7 +249,7 @@ fi
|
|||
BUILD_ARGS+=(--tag ${IMAGE_NAME} ${BUILDDIR})
|
||||
|
||||
# Build base image
|
||||
docker build "${BUILD_ARGS[@]}"
|
||||
with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed running docker build command" >&2
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2018 Wind River Systems, Inc.
|
||||
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# This utility builds the StarlingX wheel tarball
|
||||
# This utility builds the StarlingX container images
|
||||
#
|
||||
|
||||
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
|
||||
|
||||
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
|
||||
|
||||
# Required env vars
|
||||
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
|
||||
echo "Environment not setup for builds" >&2
|
||||
|
|
@ -35,6 +37,7 @@ TAG_LIST_FILE=
|
|||
TAG_LIST_LATEST_FILE=
|
||||
declare -a ONLY
|
||||
declare -a SKIP
|
||||
declare -i MAX_ATTEMPTS=1
|
||||
|
||||
function usage {
|
||||
cat >&2 <<EOF
|
||||
|
|
@ -61,6 +64,7 @@ Options:
|
|||
--skip <image> : Skip building the specified image(s). Multiple images
|
||||
can be specified with a comma-separated list, or with
|
||||
multiple --skip arguments.
|
||||
--attempts: Max attempts, in case of failure (default: 1)
|
||||
|
||||
|
||||
EOF
|
||||
|
|
@ -276,7 +280,7 @@ function build_image_loci {
|
|||
|
||||
local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}"
|
||||
|
||||
docker build ${WORKDIR}/loci --no-cache \
|
||||
with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci --no-cache \
|
||||
"${BUILD_ARGS[@]}" \
|
||||
--tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
|
||||
if [ ${PIPESTATUS[0]} -ne 0 ]; then
|
||||
|
|
@ -361,7 +365,7 @@ function build_image_docker {
|
|||
BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY)
|
||||
fi
|
||||
BASE_BUILD_ARGS+=(--tag ${build_image_name})
|
||||
docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
|
||||
with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
|
||||
|
||||
if [ ${PIPESTATUS[0]} -ne 0 ]; then
|
||||
echo "Failed to build ${LABEL}... Aborting"
|
||||
|
|
@ -395,7 +399,7 @@ function build_image {
|
|||
esac
|
||||
}
|
||||
|
||||
OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@")
|
||||
OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -477,6 +481,10 @@ while true; do
|
|||
TAG_LATEST=yes
|
||||
shift
|
||||
;;
|
||||
--attempts)
|
||||
MAX_ATTEMPTS=$2
|
||||
shift 2
|
||||
;;
|
||||
-h | --help )
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -555,6 +563,9 @@ fi
|
|||
docker images --format '{{.Repository}}:{{.Tag}}' ${BASE} | grep -q "^${BASE}$"
|
||||
BASE_IMAGE_PRESENT=$?
|
||||
|
||||
# Pull the image anyway, to ensure it's up to date
|
||||
docker pull ${BASE}
|
||||
|
||||
# Download loci, if needed.
|
||||
get_loci
|
||||
if [ $? -ne 0 ]; then
|
||||
|
|
@ -573,7 +584,7 @@ for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_
|
|||
done
|
||||
done
|
||||
|
||||
if [ "${CLEAN}" = "yes" ]; then
|
||||
if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then
|
||||
# Delete the images
|
||||
echo "Deleting images"
|
||||
docker image rm ${RESULTS_BUILT[@]} ${RESULTS_PUSHED[@]}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2018 Wind River Systems, Inc.
|
||||
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
|
@ -8,6 +8,10 @@
|
|||
# for a set of upstream python modules.
|
||||
#
|
||||
|
||||
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
|
||||
|
||||
source ${MY_SCRIPT_DIR}/utils.sh
|
||||
|
||||
# Required env vars
|
||||
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
|
||||
echo "Environment not setup for builds" >&2
|
||||
|
|
@ -21,6 +25,7 @@ OS=centos
|
|||
OS_VERSION=7.5.1804
|
||||
BUILD_STREAM=stable
|
||||
PROXY=""
|
||||
declare -i MAX_ATTEMPTS=1
|
||||
|
||||
function usage {
|
||||
cat >&2 <<EOF
|
||||
|
|
@ -34,11 +39,12 @@ Options:
|
|||
--keep-container: Skip deletion of container used for the build
|
||||
--proxy: Set proxy <URL>:<PORT>
|
||||
--stream: Build stream, stable or dev (default: stable)
|
||||
--attempts: Max attempts, in case of failure (default: 1)
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy: -- "$@")
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy:,attempts: -- "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -81,6 +87,10 @@ while true; do
|
|||
PROXY=$2
|
||||
shift 2
|
||||
;;
|
||||
--attempts)
|
||||
MAX_ATTEMPTS=$2
|
||||
shift 2
|
||||
;;
|
||||
-h | --help )
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -210,7 +220,7 @@ BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME})
|
|||
BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH})
|
||||
|
||||
# Build image
|
||||
docker build "${BUILD_ARGS[@]}"
|
||||
with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to create build image in docker" >&2
|
||||
exit 1
|
||||
|
|
@ -230,7 +240,7 @@ fi
|
|||
RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh)
|
||||
|
||||
# Run container to build wheels
|
||||
docker run ${RUN_ARGS[@]}
|
||||
with_retries ${MAX_ATTEMPTS} docker run ${RUN_ARGS[@]}
|
||||
|
||||
if [ "${KEEP_IMAGE}" = "no" ]; then
|
||||
# Delete the builder image
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2018 Wind River Systems, Inc.
|
||||
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
|
@ -9,6 +9,8 @@
|
|||
|
||||
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
|
||||
|
||||
source ${MY_SCRIPT_DIR}/utils.sh
|
||||
|
||||
# Required env vars
|
||||
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
|
||||
echo "Environment not setup for builds" >&2
|
||||
|
|
@ -25,6 +27,7 @@ PUSH=no
|
|||
PROXY=""
|
||||
CLEAN=no
|
||||
DOCKER_USER=${USER}
|
||||
declare -i MAX_ATTEMPTS=1
|
||||
|
||||
# List of top-level services for images, which should not be listed in upper-constraints.txt
|
||||
SKIP_CONSTRAINTS=(
|
||||
|
|
@ -56,11 +59,12 @@ Options:
|
|||
--proxy: Set proxy <URL>:<PORT>
|
||||
--user: Docker repo userid
|
||||
--version: Version for pushed image (if used with --push)
|
||||
--attempts: Max attempts, in case of failure (default: 1)
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version: -- "$@")
|
||||
OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version:,attempts: -- "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -111,6 +115,10 @@ while true; do
|
|||
VERSION=$2
|
||||
shift 2
|
||||
;;
|
||||
--attempts)
|
||||
MAX_ATTEMPTS=$2
|
||||
shift 2
|
||||
;;
|
||||
-h | --help )
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -143,7 +151,7 @@ if [ ! -z "$PROXY" ]; then
|
|||
BUILD_BASE_WL_ARGS+=(--proxy ${PROXY})
|
||||
fi
|
||||
|
||||
${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]}
|
||||
${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failure running build-base-wheels.sh" >&2
|
||||
exit 1
|
||||
|
|
@ -177,13 +185,13 @@ else
|
|||
OPENSTACK_BRANCH=stable/${CURRENT_STABLE_OPENSTACK}
|
||||
fi
|
||||
|
||||
wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
|
||||
with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to download global-requirements.txt" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
|
||||
with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to download upper-constraints.txt" >&2
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2018 Wind River Systems, Inc.
|
||||
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
|
@ -11,6 +11,7 @@
|
|||
CFGFILE=/wheels.cfg
|
||||
OUTPUTDIR=/wheels
|
||||
FAILED_LOG=$OUTPUTDIR/failed.lst
|
||||
declare -i MAX_ATTEMPTS=5
|
||||
|
||||
#
|
||||
# Function to log the start of a build
|
||||
|
|
@ -102,6 +103,41 @@ function fix_setup {
|
|||
|
||||
}
|
||||
|
||||
#
|
||||
# Function to call a command, with support for retries
|
||||
#
|
||||
function with_retries {
|
||||
local max_attempts=$1
|
||||
local cmd=$2
|
||||
|
||||
# Pop the first two arguments off the list,
|
||||
# so we can pass additional args to the command safely
|
||||
shift 2
|
||||
|
||||
local -i attempt=0
|
||||
|
||||
while :; do
|
||||
let -i attempt++
|
||||
|
||||
echo "Running: ${cmd} $@"
|
||||
${cmd} "$@"
|
||||
if [ $? -eq 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
|
||||
if [ ${attempt} -lt ${max_attempts} ]; then
|
||||
local delay=5
|
||||
echo "Waiting ${delay} seconds before retrying..."
|
||||
sleep ${delay}
|
||||
continue
|
||||
else
|
||||
echo "Max command attempts reached. Aborting..."
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
#
|
||||
# Function to use git to clone the module source and build a wheel.
|
||||
#
|
||||
|
|
@ -176,9 +212,8 @@ function from_tar {
|
|||
taropts="xf"
|
||||
fi
|
||||
|
||||
wget $wgetsrc
|
||||
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failure running: wget $wgetsrc"
|
||||
echo $wheelname >> $FAILED_LOG
|
||||
continue
|
||||
fi
|
||||
|
|
@ -228,9 +263,8 @@ function from_zip {
|
|||
continue
|
||||
fi
|
||||
|
||||
wget $wgetsrc
|
||||
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failure running: wget $wgetsrc"
|
||||
echo $wheelname >> $FAILED_LOG
|
||||
continue
|
||||
fi
|
||||
|
|
@ -280,9 +314,8 @@ function from_pypi {
|
|||
continue
|
||||
fi
|
||||
|
||||
wget $wgetsrc
|
||||
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failure running: wget $wgetsrc"
|
||||
echo $wheelname >> $FAILED_LOG
|
||||
continue
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Image and wheel build utility functions
|
||||
#
|
||||
|
||||
#
|
||||
# Function to call a command, with support for retries
|
||||
#
|
||||
function with_retries {
|
||||
local max_attempts=$1
|
||||
local cmd=$2
|
||||
|
||||
# Pop the first two arguments off the list,
|
||||
# so we can pass additional args to the command safely
|
||||
shift 2
|
||||
|
||||
local -i attempt=0
|
||||
|
||||
while :; do
|
||||
let -i attempt++
|
||||
|
||||
echo "Running: ${cmd} $@"
|
||||
${cmd} "$@"
|
||||
if [ $? -eq 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
|
||||
if [ ${attempt} -lt ${max_attempts} ]; then
|
||||
local delay=5
|
||||
echo "Waiting ${delay} seconds before retrying..."
|
||||
sleep ${delay}
|
||||
continue
|
||||
else
|
||||
echo "Max command attempts reached. Aborting..."
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue