starlingx/root
Code Issues Proposed changes

branching: Add retries to git push/review commands

Browse Source 
Review.opendev.org seem unreliable when hit
with a lot of transactions in a short time.
This means that an attempt to create a release branch,
e.g. r/stx.6.0 failes repeatedly, and requires a lot
of manual repair.

By adding retires and delays, we greatly improve
our odds of success.

Closes-bug: 1954536
Signed-off-by: Scott Little <scott.little@windriver.com>
Change-Id: Ia03ed58b67c02e6984a7e0ee01c6666027df6cbe
This commit is contained in:
Scott Little 2021-12-10 16:06:00 -05:00
parent d015858239
commit 06537894b2
8 changed files with 166 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
build-tools/branching/create_branches_and_tags.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
#
# Copyright (c) 2020 Wind River Systems, Inc.
# Copyright (c) 2020-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -215,18 +215,21 @@ update_gitreview () {
fi
if [ $need_commit -eq 1 ]; then
echo_stderr "determinging git review method in ${DIR}"
review_method=$(git_repo_review_method)
if [ "${review_method}" == "gerrit" ] ; then
timeout 15 git review -s
echo_stderr "running git review -s in ${DIR}"
with_retries -d 45 -t 15 -k 5 5 git review -s >&2
if [ $? != 0 ] ; then
if [ ${new_host} -eq 0 ]; then
echo_stderr "ERROR: failed to setup git review in ${DIR}"
echo_stderr "ERROR: 3: failed to setup git review in ${DIR}"
exit 1
fi
need_rm=1
message="Delete .gitreview for ${branch}"
fi
echo_stderr "finished git review -s in ${DIR}"
else
need_rm=1
message="Delete .gitreview for ${branch}"
@ -338,9 +341,9 @@ for subgit in $SUBGITS; do
review_method=$(git_repo_review_method)
if [ -f .gitreview ] && [ "${review_method}" == "gerrit" ] ; then
git review -s > /dev/null
with_retries -d 45 -t 15 -k 5 5 git review -s >&2
if [ $? != 0 ] ; then
echo_stderr "ERROR: failed to setup git review in ${subgit}"
echo_stderr "ERROR: 1: failed to setup git review in ${subgit}"
exit 1
fi
fi
@ -409,9 +412,9 @@ if [ $MANIFEST -eq 1 ]; then
review_method=$(git_review_method)
if [ -f .gitreview ] && [ "${review_method}" == "gerrit" ] ; then
git review -s > /dev/null
with_retries -d 45 -t 15 -k 5 5 git review -s >&2
if [ $? != 0 ] ; then
echo_stderr "ERROR: failed to setup git review in ${new_manifest_dir}"
echo_stderr "ERROR: 2: failed to setup git review in ${new_manifest_dir}"
exit 1
fi
fi

6
build-tools/branching/create_tags.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
#
# Copyright (c) 2020 Wind River Systems, Inc.
# Copyright (c) 2020-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -190,7 +190,7 @@ for subgit in $SUBGITS; do
review_method=$(git_repo_review_method)
if [ -f .gitreview ] && [ "${review_method}" == "gerrit" ] ; then
git review -s > /dev/null
with_retries -d 45 -t 15 -k 5 5 git review -s >&2
if [ $? != 0 ] ; then
echo_stderr "ERROR: failed to setup git review in ${subgit}"
exit 1
@ -221,7 +221,7 @@ if [ $MANIFEST -eq 1 ]; then
review_method=$(git_review_method)
if [ -f .gitreview ] && [ "${review_method}" == "gerrit" ] ; then
git review -s > /dev/null
with_retries -d 45 -t 15 -k 5 5 git review -s >&2
if [ $? != 0 ] ; then
echo_stderr "ERROR: failed to setup git review in ${new_manifest_dir}"
exit 1

52
build-tools/branching/push_branches_tags.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
#
# Copyright (c) 2020 Wind River Systems, Inc.
# Copyright (c) 2020-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -224,26 +224,25 @@ for subgit in $SUBGITS; do
port=$(url_port "${url}")
path=$(url_path "${url}")
if [ "${host}" == "review.opendev.org" ] || git_match_safe_gerrit_host "${host}" ; then
echo "git push ${review_remote} ${tag} && \\"
echo "ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \\"
echo "git config --local --replace-all branch.${branch}.merge refs/heads/${branch} && \\"
echo "git review --topic=${branch}"
git push $DRY_RUN ${review_remote} ${tag} && \
echo "git push ${review_remote} ${tag}" && \
with_retries -d 45 -t 15 -k 5 5 git push $DRY_RUN ${review_remote} ${tag} && \
echo "ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag}" && \
$DRY_RUN_CMD ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \
echo "git config --local --replace-all branch.${branch}.merge refs/heads/${branch}" && \
$DRY_RUN_CMD git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \
$DRY_RUN_CMD git review --topic="${branch}"
echo "git review --topic=${branch/\//.}" && \
$DRY_RUN_CMD with_retries -d 45 -t 15 -k 5 5 git review --topic="${branch/\//.}"
else
echo "git push ${review_remote} ${branch}:${branch} $DRY_RUN"
git push ${review_remote} ${branch}:${branch} $DRY_RUN
echo "git push ${review_remote} ${tag}:${tag} $DRY_RUN"
git push ${review_remote} ${tag}:${tag} $DRY_RUN
echo "git push ${review_remote} ${branch}:${branch} $DRY_RUN" && \
with_retries -d 45 -t 15 -k 5 5 git push ${review_remote} ${branch}:${branch} $DRY_RUN && \
echo "git push ${review_remote} ${tag}:${tag} $DRY_RUN" && \
with_retries -d 45 -t 15 -k 5 5 git push ${review_remote} ${tag}:${tag} $DRY_RUN
fi
else
echo "git push ${remote} ${branch}:${branch} $DRY_RUN"
git push ${remote} ${branch}:${branch} $DRY_RUN
echo "git push ${remote} ${tag}:${tag} $DRY_RUN"
git push ${remote} ${tag}:${tag} $DRY_RUN
echo "git push ${remote} ${branch}:${branch} $DRY_RUN" && \
with_retries -d 45 -t 15 -k 5 5 git push ${remote} ${branch}:${branch} $DRY_RUN && \
echo "git push ${remote} ${tag}:${tag} $DRY_RUN" && \
with_retries -d 45 -t 15 -k 5 5 git push ${remote} ${tag}:${tag} $DRY_RUN
fi
if [ $? != 0 ] ; then
@ -314,22 +313,21 @@ if [ $MANIFEST -eq 1 ]; then
port=$(url_port "${url}")
path=$(url_path "${url}")
if [ "${host}" == "review.opendev.org" ] || git_match_safe_gerrit_host "${host}" ; then
echo "git push ${review_remote} ${tag} && \\"
echo "ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \\"
echo "git config --local --replace-all branch.${branch}.merge refs/heads/${branch} && \\"
echo "git review --yes --topic=${branch}"
git push ${review_remote} ${tag} $DRY_RUN && \
echo "git push ${review_remote} ${tag}" && \
with_retries -d 45 -t 15 -k 5 5 git push ${review_remote} ${tag} $DRY_RUN && \
echo "ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag}" && \
$DRY_RUN_CMD ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \
echo "git config --local --replace-all branch.${branch}.merge refs/heads/${branch}" && \
$DRY_RUN_CMD git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \
$DRY_RUN_CMD git review --yes --topic="${branch}"
echo "git review --yes --topic=${branch/\//.}" && \
$DRY_RUN_CMD with_retries -d 45 -t 15 -k 5 5 git review --yes --topic="${branch/\//.}"
else
echo git push --tags ${review_remote} ${branch} $DRY_RUN
git push --tags ${review_remote} ${branch} $DRY_RUN
echo git push --tags ${review_remote} ${branch} $DRY_RUN && \
with_retries -d 45 -t 15 -k 5 5 git push --tags ${review_remote} ${branch} $DRY_RUN
fi
else
echo git push --tags --set-upstream ${review_remote} ${branch} $DRY_RUN
git push --tags --set-upstream ${review_remote} ${branch} $DRY_RUN
echo git push --tags --set-upstream ${review_remote} ${branch} $DRY_RUN && \
with_retries -d 45 -t 15 -k 5 5 git push --tags --set-upstream ${review_remote} ${branch} $DRY_RUN
fi
if [ $? != 0 ] ; then

14
build-tools/branching/push_tags.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
#
# Copyright (c) 2020 Wind River Systems, Inc.
# Copyright (c) 2020-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -182,10 +182,10 @@ for subgit in $SUBGITS; do
echo "Pushing tag $tag in ${subgit}"
if [ "${review_method}" == "gerrit" ] && [ $BYPASS_GERRIT -eq 0 ]; then
echo "git push ${review_remote} ${tag}"
git push ${review_remote} ${tag} ${DRY_RUN}
with_retries -d 45 -t 15 -k 5 5 git push ${review_remote} ${tag} ${DRY_RUN}
else
echo "git push ${remote} ${tag}"
git push ${remote} ${tag} ${DRY_RUN}
with_retries -d 45 -t 15 -k 5 5 git push ${remote} ${tag} ${DRY_RUN}
fi
if [ $? != 0 ] ; then
@ -246,7 +246,7 @@ if [ $MANIFEST -eq 1 ]; then
echo "Pushing tag $tag in ${new_manifest_dir}"
if [ "${review_method}" == "gerrit" ] && [ $BYPASS_GERRIT -eq 0 ]; then
git review
with_retries -d 45 -t 15 -k 5 5 git review
if [ $? != 0 ] ; then
echo_stderr "ERROR: Failed to create git review from ${new_manifest_dir}"
exit 1
@ -255,8 +255,10 @@ if [ $MANIFEST -eq 1 ]; then
echo " cd ${new_manifest_dir}"
echo " git push ${review_remote} ${tag}"
else
git push ${remote} ${local_branch}:${remote_branch} ${DRY_RUN}
git push ${remote} ${tag}:${tag} ${DRY_RUN}
echo "git push ${remote} ${local_branch}:${remote_branch}" && \
with_retries -d 45 -t 15 -k 5 5 git push ${remote} ${local_branch}:${remote_branch} ${DRY_RUN} && \
echo "git push ${remote} ${tag}:${tag}" && \
with_retries -d 45 -t 15 -k 5 5 git push ${remote} ${tag}:${tag} ${DRY_RUN}
fi
if [ $? != 0 ] ; then

8
build-tools/git-repo-utils.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
#
# Copyright (c) 2020 Wind River Systems, Inc.
# Copyright (c) 2020-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -182,8 +182,7 @@ git_repo_review_remote () {
if [ $? -ne 0 ]; then
# Perhaps we need to run git review -s' and try again
# echo a blank line into git in case it prompts for a username
echo | git review -s > /dev/null || return 1
with_retries -d 45 -t 15 -k 5 5 git review -s >&2 || return 1
git config remote.gerrit.url > /dev/null || return 1
fi
echo "gerrit"
@ -205,8 +204,7 @@ git_repo_review_url () {
if [ $? -ne 0 ]; then
# Perhaps we need to run git review -s' and try again
# echo a blank line into git in case it prompts for a username
echo | git review -s > /dev/null || return 1
with_retries -d 45 -t 15 -k 5 5 git review -s >&2 || return 1
git config remote.gerrit.url || return 1
fi
else

14
build-tools/git-utils.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -11,10 +11,10 @@
# by repo manifests.
#
echo_stderr ()
{
echo "$@" >&2
}
GIT_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source ${GIT_UTILS_DIR}/utils.sh
git_ctx_root_dir () {
dirname "${MY_REPO}"
@ -562,7 +562,7 @@ git_review_url () {
git config remote.gerrit.url
if [ $? -ne 0 ]; then
# Perhaps we need to run git review -s' and try again
git review -s > /dev/null || return 1
with_retries -d 45 -t 15 -k 5 5 git review -s >&2 || return 1
git config remote.gerrit.url
fi
else
@ -577,7 +577,7 @@ git_review_remote () {
git config remote.gerrit.url > /dev/null
if [ $? -ne 0 ]; then
# Perhaps we need to run git review -s' and try again
git review -s > /dev/null || return 1
with_retries -d 45 -t 15 -k 5 5 git review -s >&2 || return 1
git config remote.gerrit.url > /dev/null || return 1
fi
echo "gerrit"

12
build-tools/repo-utils.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
#
# Copyright (c) 2020 Wind River Systems, Inc.
# Copyright (c) 2020-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -10,16 +10,10 @@
# A collection of utilities relating to 'repo'
#
REPO_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
#
# Echo to stderr
# echo_stderr [any text you want]
#
source ${REPO_UTILS_DIR}/utils.sh
echo_stderr ()
{
echo "$@" >&2
}
#
# Get the root dir of a repo managed repository

107
build-tools/utils.sh Normal file
View File

@ -0,0 +1,107 @@
#!/bin/bash
#
# Copyright (c) 2019-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# Image and wheel build utility functions
#
#
# Echo to stderr
# echo_stderr [any text you want]
#
echo_stderr ()
{
echo "$@" >&2
}
#
# Function to call a command, with support for retries
#
# with_retries [<options>] <retries> <cmd> [<cmd_args>...]
#
# Options:
# -d <secs> | --delay <secs>
# Wait given number of seconds between retries
# -t <secs> | --timeout <secs>
# Each iteration of the command runs under a timeout
# -k <secs> | --kill-timeout <secs>
# Each iteration of the command is killed violently
# if it doesn't exit voluntarily within the set time
# after the initial timeout signal.
#
function with_retries {
local delay=5
local max_time=0
local kill_time=0
local to_cmd=""
while [ $1 != "" ]; do
case "$1" in
-d | --delay)
delay=$2
shift 2
;;
-t | --timeout)
max_time=$2
shift 2
;;
-k | --kill-timeout)
kill_time=$2
shift 2
;;
*)
break
;;
esac
done
local max_attempts=$1
local cmd=$2
shift 2
if [ ${max_time} -gt 0 ]; then
to_cmd="timeout "
if [ ${kill_time} -gt 0 ]; then
to_cmd+="--kill-after=${kill_time} "
fi
to_cmd+="${max_time} "
fi
# Pop the first two arguments off the list,
# so we can pass additional args to the command safely
local -i attempt=0
local rc=0
while :; do
let attempt++
echo_stderr "Running: ${cmd} $@"
${to_cmd} ${cmd} "$@"
rc=$?
if [ $rc -eq 0 ]; then
return 0
fi
if [ $rc -eq 124 ]; then
echo_stderr "Command (${cmd}) timed out, attempt ${attempt} of ${max_attempts}."
elif [ $rc -eq 137 ]; then
echo_stderr "Command (${cmd}) timed out and killed, attempt ${attempt} of ${max_attempts}."
else
echo_stderr "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
fi
if [ ${attempt} -lt ${max_attempts} ]; then
echo_stderr "Waiting ${delay} seconds before retrying..."
sleep ${delay}
continue
else
echo_stderr "Max command attempts reached. Aborting..."
return 1
fi
done
}