replace the use of zuul-cloner with a shell script
Under zuulv3 the behavior has changed significantly to now require specifying which repositories need to be cloned for each job, and zuul-cloner only copies the repositories that are already available. As soon as zuulv3 is released, our jobs relying on the old fallback behavior of checking out from a cache when present but using the upstream server in other situations will fail. The new script tools/clone_repo.sh handles the cases we have in the tools and jobs within this repository, and should also be usable from the tag-releases job. zuul is removed from the list of requirements so it is no longer installed when jobs run under tox. list_unreleased_changes.sh is updated to drop the use of the virtualenv, which was only present for zuul-cloner. The other updates in the patch are to change the tools to use the new script. Change-Id: Ic559d27881a9a89fb0dcb5295a4d7ed7c578112f Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann
parent
e611c797fc
commit
34756ddbdf
@ -31,7 +31,6 @@ import tempfile
|
||||
|
||||
import requests
|
||||
|
||||
from openstack_releases import defaults
|
||||
from openstack_releases import gitutils
|
||||
from openstack_releases import governance
|
||||
from openstack_releases import release_notes
|
||||
@ -295,26 +294,15 @@ def main():
|
||||
print('%s %s exists on git server already' %
|
||||
(project['repo'], new_release['version']))
|
||||
|
||||
# Decide which branch we're going to try to clone. We need
|
||||
# the repo checked out before we can tell if the stable
|
||||
# branch really exists, but zuul-cloner will fall back to
|
||||
# master if it doesn't.
|
||||
if series in (defaults.RELEASE, '_independent'):
|
||||
clone_branch = 'master'
|
||||
else:
|
||||
clone_branch = 'stable/' + series
|
||||
|
||||
# Check out the code.
|
||||
print('\nChecking out repository {} to {}'.format(
|
||||
project['repo'], clone_branch))
|
||||
subprocess.check_call(
|
||||
['zuul-cloner',
|
||||
'--branch', clone_branch,
|
||||
'--workspace', workdir,
|
||||
'git://git.openstack.org',
|
||||
project['repo'],
|
||||
]
|
||||
# Start by checking out master, always. We need the repo
|
||||
# checked out before we can tell if the stable branch
|
||||
# really exists.
|
||||
gitutils.clone_repo(
|
||||
workdir,
|
||||
project['repo'],
|
||||
branch='master',
|
||||
)
|
||||
|
||||
# Set some git configuration values to allow us to perform
|
||||
# local operations like tagging.
|
||||
gitutils.ensure_basic_git_config(
|
||||
@ -331,18 +319,13 @@ def main():
|
||||
else:
|
||||
branch = 'master'
|
||||
|
||||
if branch != clone_branch:
|
||||
if branch != 'master':
|
||||
# Check out the repo again to the right branch if we
|
||||
# didn't get it the first time.
|
||||
print('\nUpdating repository {} to {}'.format(
|
||||
project['repo'], branch))
|
||||
subprocess.check_call(
|
||||
['zuul-cloner',
|
||||
'--branch', branch,
|
||||
'--workspace', workdir,
|
||||
'git://git.openstack.org',
|
||||
project['repo'],
|
||||
]
|
||||
gitutils.clone_repo(
|
||||
workdir,
|
||||
project['repo'],
|
||||
branch=branch,
|
||||
)
|
||||
|
||||
# look at the previous tag for the parent of the commit
|
||||
|
||||
@ -95,34 +95,18 @@ def ensure_basic_git_config(workdir, repo, settings):
|
||||
|
||||
def clone_repo(workdir, repo, ref=None, branch=None):
|
||||
"Check out the code."
|
||||
dest = os.path.join(workdir, repo)
|
||||
if not os.path.exists(dest):
|
||||
cmd = [
|
||||
'zuul-cloner',
|
||||
'--workspace', workdir,
|
||||
]
|
||||
cache_dir = os.environ.get('ZUUL_CACHE_DIR', '/opt/git')
|
||||
if cache_dir and os.path.exists(cache_dir):
|
||||
cmd.extend(['--cache-dir', cache_dir])
|
||||
cmd.extend([
|
||||
'git://git.openstack.org',
|
||||
repo,
|
||||
])
|
||||
subprocess.check_call(cmd)
|
||||
# Force an update, just in case the local version is still out of
|
||||
# date.
|
||||
LOG.info('Updating newly cloned repository in %s' % dest)
|
||||
subprocess.check_call(
|
||||
['git', 'fetch', '-v', '--tags'],
|
||||
cwd=dest,
|
||||
)
|
||||
# If we were given some sort of reference, check that out.
|
||||
print('\nChecking out repository {} to {}'.format(
|
||||
repo, branch or ref or 'master'))
|
||||
cmd = [
|
||||
'./tools/clone_repo.sh',
|
||||
'--workspace', workdir,
|
||||
]
|
||||
if ref:
|
||||
LOG.info('Updating %s to %s' % (repo, ref))
|
||||
subprocess.check_call(
|
||||
['git', 'checkout', ref],
|
||||
cwd=dest,
|
||||
)
|
||||
cmd.extend(['--ref', ref])
|
||||
if branch:
|
||||
cmd.extend(['--branch', branch])
|
||||
cmd.append(repo)
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
|
||||
def sha_for_tag(workdir, repo, version):
|
||||
|
||||
@ -16,7 +16,6 @@ keyring==7.3
|
||||
|
||||
requests>=2.5.2
|
||||
PyYAML>=3.1.0
|
||||
zuul
|
||||
yamlordereddictloader
|
||||
prompt_toolkit
|
||||
tqdm
|
||||
|
||||
175
tools/clone_repo.sh
Executable file
175
tools/clone_repo.sh
Executable file
@ -0,0 +1,175 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# This script is used to clone a repository, possibly starting from a
|
||||
# local cache, and then update from the global remote. It replaces v2
|
||||
# of zuul-cloner, and is a bash script instead of being included in
|
||||
# openstack_releases/gitutils.py because we use it from jobs that
|
||||
# cannot run pip or tox for security reasons.
|
||||
#
|
||||
|
||||
BINDIR=$(dirname $0)
|
||||
|
||||
function print_help {
|
||||
cat <<EOF
|
||||
USAGE:
|
||||
|
||||
clone_repo.sh -h
|
||||
|
||||
clone_repo.sh [--workspace WORK_DIR] [--cache-dir CACHE]
|
||||
[--branch BRANCH] [--ref REF]
|
||||
[--upstream URL] repo-name
|
||||
|
||||
Arguments:
|
||||
|
||||
repo-name -- The full repository name, such as
|
||||
"openstack/oslo.config". This name is also used as the
|
||||
output directory.
|
||||
|
||||
Options:
|
||||
|
||||
-h -- Print help.
|
||||
|
||||
--workspace -- The name of the parent directory where the cloned
|
||||
repo should be put.
|
||||
|
||||
--cache-dir -- A location where a local copy of a repo exists and
|
||||
can be used to seed the clone, which will still be
|
||||
updated from the remote. Defaults to $ZUUL_CACHE_DIR
|
||||
or /opt/git.
|
||||
|
||||
--branch -- The branch to check out. Defaults to "master".
|
||||
|
||||
--ref -- The git reference to check out. Defaults to HEAD.
|
||||
|
||||
--upstream -- The upstream server URL, without the git repo
|
||||
part. Defaults to git://git.openstack.org
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# Defaults
|
||||
WORKSPACE="."
|
||||
CACHE_DIR="${ZUUL_CACHE_DIR:-/opt/git}"
|
||||
BRANCH="master"
|
||||
REF=""
|
||||
UPSTREAM="git://git.openstack.org"
|
||||
|
||||
OPTS=`getopt -o h --long branch:,cache-dir:,ref:,upstream:,workspace: -n $0 -- "$@"`
|
||||
if [ $? != 0 ] ; then
|
||||
echo "Failed parsing options." >&2
|
||||
print_help
|
||||
exit 1
|
||||
fi
|
||||
eval set -- "$OPTS"
|
||||
while true; do
|
||||
case "$1" in
|
||||
-h)
|
||||
print_help
|
||||
exit 0
|
||||
;;
|
||||
--branch)
|
||||
BRANCH="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
--cache-dir)
|
||||
CACHE_DIR="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
--ref)
|
||||
REF="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
--upstream)
|
||||
UPSTREAM="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
--workspace)
|
||||
WORKSPACE="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
REPO="$1"
|
||||
shift
|
||||
|
||||
if [ -z "$REPO" ]; then
|
||||
print_help
|
||||
echo "ERROR: No repository given."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$WORKSPACE" ]; then
|
||||
echo "ERROR: Workspace $WORKSPACE does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -x
|
||||
set -e
|
||||
|
||||
cache_remote="$CACHE_DIR/$REPO"
|
||||
if [ ! -d "$cache_remote" ]; then
|
||||
echo "WARNING: Cache directory $cache_remote does not exist, ignoring."
|
||||
cache_remote=""
|
||||
fi
|
||||
|
||||
upstream_remote="$UPSTREAM/$REPO"
|
||||
local_dir="$WORKSPACE/$REPO"
|
||||
|
||||
# Clone the repository.
|
||||
if [ -d $local_dir ]; then
|
||||
echo "Already have a local copy of $REPO in $local_dir"
|
||||
|
||||
elif [ ! -z "$cache_remote" ]; then
|
||||
# Clone from the cache then update the origin remote to point
|
||||
# upstream so we can pull down more recent changes.
|
||||
(cd $WORKSPACE &&
|
||||
git clone $cache_remote $REPO &&
|
||||
cd $REPO &&
|
||||
git remote set-url origin "$upstream_remote"
|
||||
)
|
||||
|
||||
else
|
||||
(cd $WORKSPACE && git clone $upstream_remote $REPO)
|
||||
fi
|
||||
|
||||
# Make sure it is up to date compared to the upstream remote.
|
||||
(cd $local_dir &&
|
||||
git remote update &&
|
||||
git fetch origin --tags
|
||||
)
|
||||
|
||||
if [ ! -z "$REF" ]; then
|
||||
# Check out the specified reference.
|
||||
(cd $local_dir && git checkout "$REF")
|
||||
else
|
||||
# Check out the expected branch (master is the default, but if the
|
||||
# directory already existed we might have checked out something else
|
||||
# before so just do it again).
|
||||
(cd $local_dir &&
|
||||
git checkout $BRANCH &&
|
||||
git pull --ff-only)
|
||||
fi
|
||||
@ -101,15 +101,15 @@ function update_upper_constraints {
|
||||
}
|
||||
|
||||
|
||||
_functions_bindir=$(realpath $(dirname $0))
|
||||
|
||||
function clone_repo {
|
||||
typeset repo="$1"
|
||||
typeset branch="$2"
|
||||
if [ -z "$branch" ]; then
|
||||
branch="master"
|
||||
fi
|
||||
output=$({ zuul-cloner --branch "$branch" git://git.openstack.org $repo \
|
||||
&& (cd $repo && git remote -v update \
|
||||
&& git fetch -v --tags); } 2>&1)
|
||||
output=$($_functions_bindir/clone_repo.sh --branch "$branch" $repo 2>&1)
|
||||
_retval=$?
|
||||
if [ $_retval -ne 0 ] ; then
|
||||
echo "$output"
|
||||
|
||||
@ -28,13 +28,6 @@ TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
BASEDIR=$(dirname $TOOLSDIR)
|
||||
source $TOOLSDIR/functions
|
||||
|
||||
if [[ -z "$VIRTUAL_ENV" ]]; then
|
||||
if [[ ! -d $BASEDIR/.tox/venv ]]; then
|
||||
(cd $BASEDIR && tox -e venv --notest)
|
||||
fi
|
||||
source $BASEDIR/.tox/venv/bin/activate
|
||||
fi
|
||||
|
||||
# Make sure no pager is configured so the output is not blocked
|
||||
export PAGER=
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user