starlingx/root
Code Issues Proposed changes

Merge "Retries for certain wheel and image build commands"

Browse Source
This commit is contained in:
Zuul 2019-04-11 19:48:49 +00:00 committed by Gerrit Code Review
commit c88c281fa1
6 changed files with 143 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
build-tools/build-docker-images/build-stx-base.sh
View File

@ -1,14 +1,16 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This utility builds the StarlingX wheel tarball
# This utility builds the StarlingX base image
#
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
@ -31,6 +33,7 @@ CLEAN=no
TAG_LATEST=no
LATEST_TAG=latest
HOST=${HOSTNAME}
declare -i MAX_ATTEMPTS=1
function usage {
cat >&2 <<EOF
@ -52,11 +55,12 @@ Options:
--registry: Docker registry
--clean: Remove image(s) from local registry
--hostname: build repo host
--attempts: Max attempts, in case of failure (default: 1)
EOF
}
OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@")
OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname:,attempts: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -132,6 +136,10 @@ while true; do
HOST=$2
shift 2
;;
--attempts)
MAX_ATTEMPTS=$2
shift 2
;;
-h | --help )
usage
exit 1
@ -223,6 +231,9 @@ done
docker images --format '{{.Repository}}:{{.Tag}}' ${OS}:${OS_VERSION} | grep -q "^${OS}:${OS_VERSION}$"
BASE_IMAGE_PRESENT=$?
# Pull the image anyway, to ensure it is up to date
docker pull ${OS}:${OS_VERSION}
# Build the image
IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION}
IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG}
@ -238,7 +249,7 @@ fi
BUILD_ARGS+=(--tag ${IMAGE_NAME} ${BUILDDIR})
# Build base image
docker build "${BUILD_ARGS[@]}"
with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
if [ $? -ne 0 ]; then
echo "Failed running docker build command" >&2
exit 1

23
build-tools/build-docker-images/build-stx-images.sh
View File

@ -1,14 +1,16 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This utility builds the StarlingX wheel tarball
# This utility builds the StarlingX container images
#
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
@ -35,6 +37,7 @@ TAG_LIST_FILE=
TAG_LIST_LATEST_FILE=
declare -a ONLY
declare -a SKIP
declare -i MAX_ATTEMPTS=1
function usage {
cat >&2 <<EOF
@ -61,6 +64,7 @@ Options:
--skip <image> : Skip building the specified image(s). Multiple images
can be specified with a comma-separated list, or with
multiple --skip arguments.
--attempts: Max attempts, in case of failure (default: 1)
EOF
@ -276,7 +280,7 @@ function build_image_loci {
local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}"
docker build ${WORKDIR}/loci --no-cache \
with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci --no-cache \
"${BUILD_ARGS[@]}" \
--tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
if [ ${PIPESTATUS[0]} -ne 0 ]; then
@ -361,7 +365,7 @@ function build_image_docker {
BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY)
fi
BASE_BUILD_ARGS+=(--tag ${build_image_name})
docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
if [ ${PIPESTATUS[0]} -ne 0 ]; then
echo "Failed to build ${LABEL}... Aborting"
@ -395,7 +399,7 @@ function build_image {
esac
}
OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@")
OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -477,6 +481,10 @@ while true; do
TAG_LATEST=yes
shift
;;
--attempts)
MAX_ATTEMPTS=$2
shift 2
;;
-h | --help )
usage
exit 1
@ -555,6 +563,9 @@ fi
docker images --format '{{.Repository}}:{{.Tag}}' ${BASE} | grep -q "^${BASE}$"
BASE_IMAGE_PRESENT=$?
# Pull the image anyway, to ensure it's up to date
docker pull ${BASE}
# Download loci, if needed.
get_loci
if [ $? -ne 0 ]; then
@ -573,7 +584,7 @@ for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_
done
done
if [ "${CLEAN}" = "yes" ]; then
if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then
# Delete the images
echo "Deleting images"
docker image rm ${RESULTS_BUILT[@]} ${RESULTS_PUSHED[@]}

18
build-tools/build-wheels/build-base-wheels.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -8,6 +8,10 @@
# for a set of upstream python modules.
#
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
@ -21,6 +25,7 @@ OS=centos
OS_VERSION=7.5.1804
BUILD_STREAM=stable
PROXY=""
declare -i MAX_ATTEMPTS=1
function usage {
cat >&2 <<EOF
@ -34,11 +39,12 @@ Options:
--keep-container: Skip deletion of container used for the build
--proxy: Set proxy <URL>:<PORT>
--stream: Build stream, stable or dev (default: stable)
--attempts: Max attempts, in case of failure (default: 1)
EOF
}
OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy: -- "$@")
OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy:,attempts: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -81,6 +87,10 @@ while true; do
PROXY=$2
shift 2
;;
--attempts)
MAX_ATTEMPTS=$2
shift 2
;;
-h | --help )
usage
exit 1
@ -210,7 +220,7 @@ BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME})
BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH})
# Build image
docker build "${BUILD_ARGS[@]}"
with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
if [ $? -ne 0 ]; then
echo "Failed to create build image in docker" >&2
exit 1
@ -230,7 +240,7 @@ fi
RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh)
# Run container to build wheels
docker run ${RUN_ARGS[@]}
with_retries ${MAX_ATTEMPTS} docker run ${RUN_ARGS[@]}
if [ "${KEEP_IMAGE}" = "no" ]; then
# Delete the builder image

18
build-tools/build-wheels/build-wheel-tarball.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -9,6 +9,8 @@
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
@ -25,6 +27,7 @@ PUSH=no
PROXY=""
CLEAN=no
DOCKER_USER=${USER}
declare -i MAX_ATTEMPTS=1
# List of top-level services for images, which should not be listed in upper-constraints.txt
SKIP_CONSTRAINTS=(
@ -56,11 +59,12 @@ Options:
--proxy: Set proxy <URL>:<PORT>
--user: Docker repo userid
--version: Version for pushed image (if used with --push)
--attempts: Max attempts, in case of failure (default: 1)
EOF
}
OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version: -- "$@")
OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version:,attempts: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -111,6 +115,10 @@ while true; do
VERSION=$2
shift 2
;;
--attempts)
MAX_ATTEMPTS=$2
shift 2
;;
-h | --help )
usage
exit 1
@ -143,7 +151,7 @@ if [ ! -z "$PROXY" ]; then
BUILD_BASE_WL_ARGS+=(--proxy ${PROXY})
fi
${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]}
${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS}
if [ $? -ne 0 ]; then
echo "Failure running build-base-wheels.sh" >&2
exit 1
@ -177,13 +185,13 @@ else
OPENSTACK_BRANCH=stable/${CURRENT_STABLE_OPENSTACK}
fi
wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
if [ $? -ne 0 ]; then
echo "Failed to download global-requirements.txt" >&2
exit 1
fi
wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
if [ $? -ne 0 ]; then
echo "Failed to download upper-constraints.txt" >&2
exit 1

47
build-tools/build-wheels/docker/docker-build-wheel.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -11,6 +11,7 @@
CFGFILE=/wheels.cfg
OUTPUTDIR=/wheels
FAILED_LOG=$OUTPUTDIR/failed.lst
declare -i MAX_ATTEMPTS=5
#
# Function to log the start of a build
@ -102,6 +103,41 @@ function fix_setup {
}
#
# Function to call a command, with support for retries
#
function with_retries {
local max_attempts=$1
local cmd=$2
# Pop the first two arguments off the list,
# so we can pass additional args to the command safely
shift 2
local -i attempt=0
while :; do
let -i attempt++
echo "Running: ${cmd} $@"
${cmd} "$@"
if [ $? -eq 0 ]; then
return 0
fi
echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
if [ ${attempt} -lt ${max_attempts} ]; then
local delay=5
echo "Waiting ${delay} seconds before retrying..."
sleep ${delay}
continue
else
echo "Max command attempts reached. Aborting..."
return 1
fi
done
}
#
# Function to use git to clone the module source and build a wheel.
#
@ -176,9 +212,8 @@ function from_tar {
taropts="xf"
fi
wget $wgetsrc
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
if [ $? -ne 0 ]; then
echo "Failure running: wget $wgetsrc"
echo $wheelname >> $FAILED_LOG
continue
fi
@ -228,9 +263,8 @@ function from_zip {
continue
fi
wget $wgetsrc
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
if [ $? -ne 0 ]; then
echo "Failure running: wget $wgetsrc"
echo $wheelname >> $FAILED_LOG
continue
fi
@ -280,9 +314,8 @@ function from_pypi {
continue
fi
wget $wgetsrc
with_retries ${MAX_ATTEMPTS} wget $wgetsrc
if [ $? -ne 0 ]; then
echo "Failure running: wget $wgetsrc"
echo $wheelname >> $FAILED_LOG
continue
fi

44
build-tools/build-wheels/utils.sh Normal file
View File

@ -0,0 +1,44 @@
#!/bin/bash
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# Image and wheel build utility functions
#
#
# Function to call a command, with support for retries
#
function with_retries {
local max_attempts=$1
local cmd=$2
# Pop the first two arguments off the list,
# so we can pass additional args to the command safely
shift 2
local -i attempt=0
while :; do
let -i attempt++
echo "Running: ${cmd} $@"
${cmd} "$@"
if [ $? -eq 0 ]; then
return 0
fi
echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
if [ ${attempt} -lt ${max_attempts} ]; then
local delay=5
echo "Waiting ${delay} seconds before retrying..."
sleep ${delay}
continue
else
echo "Max command attempts reached. Aborting..."
return 1
fi
done
}