diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index 441b8d1d..ebfddb8c 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -9,7 +9,7 @@ MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) -source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh +source ${MY_SCRIPT_DIR}/../utils.sh # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then @@ -38,6 +38,7 @@ LATEST_TAG=latest HOST=${HOSTNAME} USE_DOCKER_CACHE=no declare -i MAX_ATTEMPTS=1 +declare -i RETRY_DELAY=30 function usage { cat >&2 < + Max attempts, in case of failure (default: 1) + --retry-delay + Sleep this many seconds between retries (default: 30) --config-file:Specify a path to a config file which will specify additional arguments to be passed into the command --cache: Allow docker to use cached filesystem layers when building @@ -118,7 +122,7 @@ function get_args_from_file { done <"$1" } -OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,cache,hostname:,attempts:,config-file: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,cache,hostname:,attempts:,retry-delay:,config-file: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -202,6 +206,10 @@ while true; do MAX_ATTEMPTS=$2 shift 2 ;; + --retry-delay) + RETRY_DELAY=$2 + shift 2 + ;; --config-file) CONFIG_FILE=$2 shift 2 @@ -404,7 +412,7 @@ fi BUILD_ARGS+=(--tag ${IMAGE_NAME} ${BUILDDIR}) # Build base image -with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}" +with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}" if [ $? -ne 0 ]; then echo "Failed running docker build command" >&2 exit 1 @@ -413,7 +421,7 @@ fi if [ "${PUSH}" = "yes" ]; then # Push the image echo "Pushing image: ${IMAGE_NAME}" - docker push ${IMAGE_NAME} + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker push ${IMAGE_NAME} if [ $? -ne 0 ]; then echo "Failed running docker push command" >&2 exit 1 @@ -422,7 +430,7 @@ if [ "${PUSH}" = "yes" ]; then if [ "$TAG_LATEST" = "yes" ]; then docker tag ${IMAGE_NAME} ${IMAGE_NAME_LATEST} echo "Pushing image: ${IMAGE_NAME_LATEST}" - docker push ${IMAGE_NAME_LATEST} + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker push ${IMAGE_NAME_LATEST} if [ $? -ne 0 ]; then echo "Failed running docker push command on latest" >&2 exit 1 diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index 9274a1ce..c4f11efa 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -9,8 +9,6 @@ MY_SCRIPT_DIR=$(dirname $(readlink -fv $0)) -source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh - # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then echo "Environment not setup for builds" >&2 @@ -45,6 +43,7 @@ DEFAULT_SPICE_REPO="https://gitlab.freedesktop.org/spice/spice-html5" declare -a ONLY declare -a SKIP declare -i MAX_ATTEMPTS=1 +declare -i RETRY_DELAY=30 declare -a RESULTS_BUILT declare -a RESULTS_PUSHED @@ -89,7 +88,10 @@ Options: --skip : Skip building the specified image(s). Multiple images can be specified with a comma-separated list, or with multiple --skip arguments. - --attempts: Max attempts, in case of failure (default: 1) + --attempts + How many times to try a failed build command (default: 1) + --retry-delay + Sleep this many seconds between retries (default: 30) --cache: Allow docker to use filesystem cache when building CAUTION: this option may ignore locally-generated @@ -180,11 +182,13 @@ function get_git { if [ ! -d ${WORKDIR}/${git_name} ]; then cd ${WORKDIR} - git clone --recursive ${git_repo} + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} $SHELL -c "rm -rf ${git_name}.clone_tmp && git clone --recursive ${git_repo} ${git_name}.clone_tmp" if [ $? -ne 0 ]; then + rm -rf ${git_name}.clone_tmp echo "Failed to clone ${git_repo}. Aborting..." >&2 return 1 fi + mv ${git_name}.clone_tmp ${git_name} cd $git_name git checkout ${git_ref} @@ -206,7 +210,7 @@ function get_git { else cd ${WORKDIR}/${git_name} - git fetch + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} git fetch if [ $? -ne 0 ]; then echo "Failed to fetch '${git_name}'. Aborting..." >&2 return 1 @@ -347,7 +351,7 @@ function post_build { local push_tag="${DOCKER_REGISTRY}${DOCKER_USER}/${LABEL}:${IMAGE_TAG_VERSIONED}" docker tag ${build_image_name} ${push_tag} - with_retries ${MAX_ATTEMPTS} docker push ${push_tag} + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker push ${push_tag} if [[ $? -ne 0 ]] ; then echo "Failed to push ${push_tag} ... Aborting" RESULTS_PUSH_FAILED+=(${LABEL}) @@ -361,7 +365,7 @@ function post_build { if [ "$TAG_LATEST" = "yes" ]; then local latest_tag="${DOCKER_REGISTRY}${DOCKER_USER}/${LABEL}:${IMAGE_TAG_LATEST}" docker tag ${push_tag} ${latest_tag} - with_retries ${MAX_ATTEMPTS} docker push ${latest_tag} + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker push ${latest_tag} if [[ $? -ne 0 ]] ; then echo "Failed to push ${latest_tag} ... Aborting" RESULTS_PUSH_FAILED+=(${LABEL}) @@ -457,7 +461,8 @@ function build_image_loci { echo "Creating bare clone of ${PROJECT_REPO} for ${LABEL} build..." if [ -n "${PROJECT_REF}" ]; then echo "PROJECT_REF specified is ${PROJECT_REF}..." - git clone --no-local --bare ${PROJECT_REPO} ${CLONE_DIR} \ + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} $SHELL -c "rm -rf ${CLONE_DIR}.clone_tmp && git clone --no-local --bare ${PROJECT_REPO} ${CLONE_DIR}.clone_tmp" \ + && mv ${CLONE_DIR}.clone_tmp ${CLONE_DIR} \ && cd ${PROJECT_REPO} \ && git push --force ${CLONE_DIR} HEAD:refs/heads/${PROJECT_REF} \ && mv ${CLONE_DIR}/hooks/post-update.sample ${CLONE_DIR}/hooks/post-update \ @@ -466,7 +471,8 @@ function build_image_loci { && git update-server-info \ && cd ${ORIGWD} else - git clone --no-local --bare ${PROJECT_REPO} ${CLONE_DIR} \ + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} $SHELL -c "rm -rf ${CLONE_DIR}.clone_tmp && git clone --no-local --bare ${PROJECT_REPO} ${CLONE_DIR}.clone_tmp" \ + && mv ${CLONE_DIR}.clone_tmp ${CLONE_DIR} \ && cd ${PROJECT_REPO} \ && mv ${CLONE_DIR}/hooks/post-update.sample ${CLONE_DIR}/hooks/post-update \ && chmod a+x ${CLONE_DIR}/hooks/post-update \ @@ -476,6 +482,7 @@ function build_image_loci { fi if [ $? -ne 0 ]; then + rm -rf ${CLONE_DIR}.clone_tmp echo "Failed to clone ${PROJECT_REPO}... Aborting ${LABEL} build" RESULTS_FAILED+=(${LABEL}) cd ${ORIGWD} @@ -571,7 +578,7 @@ function build_image_loci { local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}" - with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci \ + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci \ "${BUILD_ARGS[@]}" \ --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS_LABEL}-${BUILD_STREAM}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then @@ -704,7 +711,7 @@ function build_image_docker { fi BASE_BUILD_ARGS+=(--tag ${build_image_name}) - with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS_LABEL}-${BUILD_STREAM}.log + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS_LABEL}-${BUILD_STREAM}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then echo "Failed to build ${LABEL}... Aborting" @@ -775,7 +782,7 @@ function build_image_script { cp $(dirname ${image_build_file})/${SCRIPT} ${SCRIPT} local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}" - with_retries ${MAX_ATTEMPTS} ${COMMAND} ${SCRIPT} ${ARGS} ${build_image_name} $HTTP_PROXY $HTTPS_PROXY $NO_PROXY 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS_LABEL}-${BUILD_STREAM}.log + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} ${COMMAND} ${SCRIPT} ${ARGS} ${build_image_name} $HTTP_PROXY $HTTPS_PROXY $NO_PROXY 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS_LABEL}-${BUILD_STREAM}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then echo "Failed to build ${LABEL}... Aborting" @@ -817,7 +824,7 @@ function build_image { esac } -OPTS=$(getopt -o hN -l help,os:,os-label:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,wheels-py2:,only:,skip:,prefix:,latest,latest-prefix:,clean,cache,attempts:,no-pull-base -- "$@") +OPTS=$(getopt -o hN -l help,os:,os-label:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,wheels-py2:,only:,skip:,prefix:,latest,latest-prefix:,clean,cache,attempts:,retry-delay:,no-pull-base -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -923,6 +930,10 @@ while true; do MAX_ATTEMPTS=$2 shift 2 ;; + --retry-delay) + RETRY_DELAY=$2 + shift 2 + ;; -N|--no-pull-base) PULL_BASE=no shift @@ -1105,7 +1116,7 @@ BASE_IMAGE_PRESENT=$? # Pull the image anyway, to ensure it's up to date if [[ "$PULL_BASE" == "yes" ]] ; then - docker pull ${BASE} + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker pull ${BASE} || exit 1 fi # Download loci, if needed. diff --git a/build-tools/build-docker-images/update-stx-image.sh b/build-tools/build-docker-images/update-stx-image.sh index ff7ea8f7..da6cd3d2 100755 --- a/build-tools/build-docker-images/update-stx-image.sh +++ b/build-tools/build-docker-images/update-stx-image.sh @@ -9,7 +9,7 @@ MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) -source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh +source ${MY_SCRIPT_DIR}/../utils.sh # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then @@ -35,6 +35,7 @@ declare -a DIST_PACKAGES declare -a MODULE_SRC declare -a EXTRA_FILES declare -i MAX_ATTEMPTS=1 +declare -i RETRY_DELAY=30 function usage { @@ -67,7 +68,8 @@ Options: --user: Docker repo userid --registry: Docker registry --clean: Remove image(s) from local registry - --attempts: Max attempts, in case of failure (default: 1) + --attempts : Max attempts, in case of failure (default: 1) + --retry-delay : Sleep between retries (default: 30) --update-id: Update ID @@ -95,7 +97,7 @@ function copy_files_to_workdir { for f in $*; do if [[ ${f} =~ ^(http|https|git): ]]; then pushd ${destdir} - with_retries ${MAX_ATTEMPTS} wget ${f} + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} wget ${f} if [ $? -ne 0 ]; then echo "Failed to download $f to ${destdir}" >&2 exit 1 @@ -188,7 +190,7 @@ function read_params_from_file { FILE_BASEDIR=$(dirname ${FILE}) } -OPTS=$(getopt -o h -l help,file:,from:,wheel:,module-src:,pkg:,customize:,extra:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,clean,attempts:,update-id: -- "$@") +OPTS=$(getopt -o h -l help,file:,from:,wheel:,module-src:,pkg:,customize:,extra:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,clean,attempts:,retry-delay:,update-id: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -268,6 +270,10 @@ while true; do MAX_ATTEMPTS=$2 shift 2 ;; + --retry-delay) + RETRY_DELAY=$2 + shift 2 + ;; --update-id) UPDATE_ID=$2 shift 2 diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index 899e82ef..0199fa94 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -10,7 +10,7 @@ MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) -source ${MY_SCRIPT_DIR}/utils.sh +source ${MY_SCRIPT_DIR}/../utils.sh # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then @@ -30,6 +30,7 @@ NO_PROXY="" USE_DOCKER_CACHE=no : ${PYTHON3:=python3} declare -i MAX_ATTEMPTS=1 +declare -i RETRY_DELAY=30 function usage { cat >&2 <:, urls splitted by "," --no_proxy: Set bypass list for proxy , urls splitted by "," --stream: Build stream, stable or dev (default: stable) - --attempts: Max attempts, in case of failure (default: 1) + --attempts + Max attempts, in case of failure (default: 1) + --retry-delay + Sleep this many seconds between retries (default: 30) --cache: Allow docker to use filesystem cache when building CAUTION: this option may ignore locally-generated @@ -55,7 +59,7 @@ Options: EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,http_proxy:,https_proxy:,no_proxy:,attempts:,cache -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,http_proxy:,https_proxy:,no_proxy:,attempts:,retry-delay:,cache -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -110,6 +114,10 @@ while true; do MAX_ATTEMPTS=$2 shift 2 ;; + --retry-delay) + RETRY_DELAY=$2 + shift 2 + ;; --cache) USE_DOCKER_CACHE=yes shift @@ -238,7 +246,7 @@ if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then docker images --format '{{.Repository}}:{{.Tag}}' ${MASTER_WHEELS_IMAGE} | grep -q "^${MASTER_WHEELS_IMAGE}$" MASTER_WHEELS_PRESENT=$? - docker pull ${MASTER_WHEELS_IMAGE} + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker pull ${MASTER_WHEELS_IMAGE} if [ $? -ne 0 ]; then echo "Failed to pull ${MASTER_WHEELS_IMAGE}" >&2 exit 1 @@ -378,7 +386,7 @@ BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME}) BUILD_ARGS+=(-f ${DOCKER_BUILD_PATH}/${OS}/Dockerfile ${DOCKER_BUILD_PATH}) # Build image -with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}" +with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}" if [ $? -ne 0 ]; then echo "Failed to create build image in docker" >&2 exit 1 @@ -407,13 +415,13 @@ rm -f ${BUILD_OUTPUT_PATH_PY2}/failed.lst notice "building python3 wheels" log_prefix "[python3] " \ - with_retries ${MAX_ATTEMPTS} \ + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} \ docker run ${RUN_ARGS[@]} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh BUILD_STATUS=$? notice "building python2 wheels" log_prefix "[python2] " \ - with_retries ${MAX_ATTEMPTS} \ + with_retries -d ${RETRY_DELAY} ${MAX_ATTEMPTS} \ docker run ${RUN_ARGS[@]} -v ${BUILD_OUTPUT_PATH_PY2}:/wheels --env PYTHON=python2 ${BUILD_IMAGE_NAME} /docker-build-wheel.sh BUILD_STATUS_PY2=$? diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index d65f7bc7..8324fd13 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -9,7 +9,7 @@ MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) -source ${MY_SCRIPT_DIR}/utils.sh +source ${MY_SCRIPT_DIR}/../utils.sh # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then @@ -30,6 +30,7 @@ CLEAN=no KEEP_IMAGE=no DOCKER_USER=${USER} declare -i MAX_ATTEMPTS=1 +declare -i RETRY_DELAY=30 PYTHON2=no USE_DOCKER_CACHE=no EXTRA_WHEELS_DIR= @@ -64,28 +65,29 @@ Usage: $(basename $0) Options: - --os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]}) - --os-version: Specify OS version - --stream: Build stream, stable or dev (default: stable) - --push: Push to docker repo - --http_proxy: Set http proxy :, urls splitted by "," + --os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]}) + --os-version: Specify OS version + --stream: Build stream, stable or dev (default: stable) + --push: Push to docker repo + --http_proxy: Set http proxy :, urls splitted by "," --https_proxy: Set https proxy :, urls splitted by "," - --no_proxy: Set bypass list for proxy , urls splitted by "," - --user: Docker repo userid - --version: Version for pushed image (if used with --push) - --attempts: Max attempts, in case of failure (default: 1) - --python2: Build a python2 tarball + --no_proxy: Set bypass list for proxy , urls splitted by "," + --user: Docker repo userid + --version: Version for pushed image (if used with --push) + --attempts: Max attempts, in case of failure (default: 1) + --retry-delay: Sleep this many seconds between retries (default: 30) + --python2: Build a python2 tarball --extra-wheels-dir: Directory containing additional .whl files - --keep-image: Don't delete wheel builder image at the end + --keep-image: Don't delete wheel builder image at the end - --cache: Allow docker to use filesystem cache when building - CAUTION: this option may ignore locally-generated - packages and is meant for debugging the build - scripts. + --cache: Allow docker to use filesystem cache when building + CAUTION: this option may ignore locally-generated + packages and is meant for debugging the build + scripts. EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts:,python2,extra-wheels-dir:,keep-image,cache -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts:,retry-delay:,python2,extra-wheels-dir:,keep-image,cache -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -148,6 +150,10 @@ while true; do MAX_ATTEMPTS=$2 shift 2 ;; + --retry-delay) + RETRY_DELAY=$2 + shift 2 + ;; --python2) PYTHON2=yes shift @@ -238,7 +244,7 @@ if [[ "$USE_DOCKER_CACHE" == "yes" ]] ; then BUILD_BASE_WL_ARGS+=(--cache) fi -${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS} +${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts "${MAX_ATTEMPTS}" --retry-delay "${RETRY_DELAY}" if [ $? -ne 0 ]; then echo "Failure running build-base-wheels.sh" >&2 exit 1 diff --git a/build-tools/build-wheels/utils.sh b/build-tools/build-wheels/utils.sh deleted file mode 100644 index 370e206c..00000000 --- a/build-tools/build-wheels/utils.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2019 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Image and wheel build utility functions -# - -# -# Function to call a command, with support for retries -# -function with_retries { - local max_attempts=$1 - local cmd=$2 - - # Pop the first two arguments off the list, - # so we can pass additional args to the command safely - shift 2 - - local -i attempt=0 - - while :; do - let attempt++ - - echo "Running: ${cmd} $@" - ${cmd} "$@" - if [ $? -eq 0 ]; then - return 0 - fi - - echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}." - if [ ${attempt} -lt ${max_attempts} ]; then - local delay=5 - echo "Waiting ${delay} seconds before retrying..." - sleep ${delay} - continue - else - echo "Max command attempts reached. Aborting..." - return 1 - fi - done -} -