Merge "Add release scripts build-context.py and branch-repo.sh"
This commit is contained in:
Zuul
Gerrit Code Review
commit
86d14b6200
|
|
@ -0,0 +1,37 @@
|
|||
=============
|
||||
Release Tools
|
||||
=============
|
||||
|
||||
A set of tools used in the StarlingX release process past, current and future...
|
||||
|
||||
``branch-repo.sh`` - Derived from the older ``branch-stx.sh`` to support
|
||||
reading a list of repos, paths and SHAs from stdin, usually the output of
|
||||
``build-context.py``. This simplifies the creation of branching based on
|
||||
a specific build using the CONTEXT.sh file generated by the build system.
|
||||
|
||||
``build-context.py`` - Takes a manifest file and an optional CONTEXT.sh file
|
||||
to produce a list of repositories and their paths and SHAs that represent a
|
||||
specific build. This can be fed directly into ``branch-repo.sh`` to create
|
||||
branches corresponding to that specific build. (This is in Python completely
|
||||
due to combining XML and the joys of dict-like manipulation in shell.)
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
Here is an example of creating a new branch named 'r/stx.2.0' and tag 'v2.0.0.rc0' from
|
||||
the lastest green build context at the time:
|
||||
|
||||
::
|
||||
|
||||
release/build-context.py \
|
||||
--remote starlingx \
|
||||
--context http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/latest_green_build/outputs/CONTEXT.sh \
|
||||
https://opendev.org/starlingx/manifest/raw/branch/master/default.xml | \
|
||||
release/branch-repo.sh -b r/stx.2.0 -t v2.0.0.rc0
|
||||
|
||||
Older Scripts
|
||||
=============
|
||||
|
||||
branch-stx.sh - Shell script previously used to create release and milestone branches. This has a number of assumptions baked in regarding branch and tag names.
|
||||
|
||||
get-repo.sh - Called from ``branch-stx.sh`` to read the XML manifest and extract repo names from specific remotes. This has totally been absorbed into ``build-context.py``.
|
||||
|
|
@ -0,0 +1,205 @@
|
|||
#!/bin/bash
|
||||
# branch-repo.sh - Create new branches in a set of Git repositories
|
||||
#
|
||||
# branch-repo.sh [--dry-run|-n] [-b <branch>] [-t <tag>] [-s <source-branch>] [-i]
|
||||
#
|
||||
# --dry-run|-n Do all work except pushing back to the remote repo.
|
||||
# Useful to validate everything locally before pushing.
|
||||
#
|
||||
# -b <branch> Name of the new branch
|
||||
#
|
||||
# -t <tag> Apply a tag at the SHA passed in the input data, or HEAD
|
||||
# of the new branch if no SHA is present
|
||||
#
|
||||
# -s <source-branch> The starting branch to use instead of the default 'master'.
|
||||
# This is needed when the working branch is not named 'master'.
|
||||
# Setting <source-branch> == <branch> makes this a tag-only
|
||||
# operation (.gitreview updates are skipped).
|
||||
#
|
||||
# -i Ignore path in input; use the last component of the repo
|
||||
# name for the path similar to git clone's default.
|
||||
#
|
||||
# Read a list of repo tuples from stdin:
|
||||
# <url> <local-path> <sha>
|
||||
#
|
||||
# For each repo:
|
||||
# * create a new branch <branch> at <sha>, or at HEAD of SRC_BRANCH if no <sha>
|
||||
# * tag the new branch with an initial release identifier if <tag> is set
|
||||
# * update the .gitreview file to default to the new branch (Gerrit repos only)
|
||||
#
|
||||
# Some environment variables are available for modifying this script's behaviour
|
||||
# NOTE: The command-line options override the environment variables when
|
||||
# both are present.
|
||||
#
|
||||
# - BRANCH sets the new branch name <branch>
|
||||
#
|
||||
# - SRC_BRANCH sets the source branch name <source-branch>.
|
||||
#
|
||||
# - TAG sets the release tag <tag>.
|
||||
#
|
||||
# More Notes
|
||||
# * The detection to use Gerrit or Github is determined by the presence of
|
||||
# 'git.starlingx.io' or 'opendev.org' in the repo URL. This may be
|
||||
# sub-optimal. The only actual difference in execution is .gitreview
|
||||
# updates are only prepared for Gerrit repos.
|
||||
|
||||
set -e
|
||||
|
||||
# Defaults
|
||||
BRANCH=${BRANCH:-""}
|
||||
SRC_BRANCH=${SRC_BRANCH:-master}
|
||||
TAG=${TAG:-""}
|
||||
|
||||
optspec="b:ins:t:-:"
|
||||
while getopts "$optspec" o; do
|
||||
case "${o}" in
|
||||
# Hack in longopt support
|
||||
-)
|
||||
case "${OPTARG}" in
|
||||
dry-run)
|
||||
DRY_RUN=1
|
||||
;;
|
||||
*)
|
||||
if [[ "$OPTERR" = 1 ]] && [[ "${optspec:0:1}" != ":" ]]; then
|
||||
echo "Unknown option --${OPTARG}" >&2
|
||||
fi
|
||||
;;
|
||||
|
||||
esac
|
||||
;;
|
||||
b)
|
||||
BRANCH=${OPTARG}
|
||||
;;
|
||||
i)
|
||||
SKIP_PATH=1
|
||||
;;
|
||||
n)
|
||||
DRY_RUN=1
|
||||
;;
|
||||
s)
|
||||
SRC_BRANCH=${OPTARG}
|
||||
;;
|
||||
t)
|
||||
TAG=${OPTARG}
|
||||
;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND-1))
|
||||
|
||||
# See if we can build a repo list
|
||||
if [[ -z $BRANCH ]]; then
|
||||
echo "ERROR: No repos to process"
|
||||
echo "Usage: $0 [--dry-run|-n] [-b <branch>] [-t <tag>] [-s <source-branch>] [-i]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# This is where other scripts live that we need
|
||||
script_dir=$(realpath $(dirname $0))
|
||||
|
||||
# update_gitreview <branch>
|
||||
# Based on update_gitreview() from https://github.com/openstack/releases/blob/a7db6cf156ba66d50e1955db2163506365182ee8/tools/functions#L67
|
||||
function update_gitreview {
|
||||
typeset branch="$1"
|
||||
|
||||
git checkout $branch
|
||||
# Remove a trailing newline, if present, to ensure consistent
|
||||
# formatting when we add the defaultbranch line next.
|
||||
typeset grcontents="$(echo -n "$(cat .gitreview | grep -v defaultbranch)")
|
||||
defaultbranch=$branch"
|
||||
echo "$grcontents" > .gitreview
|
||||
git add .gitreview
|
||||
if git commit -s -m "Update .gitreview for $branch"; then
|
||||
if [[ -z $DRY_RUN ]]; then
|
||||
git review -t "create-${branch}"
|
||||
else
|
||||
echo "### skipping .gitreview submission to $branch"
|
||||
fi
|
||||
else
|
||||
echo "### no changes required for .gitreview"
|
||||
fi
|
||||
}
|
||||
|
||||
# branch_repo <repo-uri> <path> <sha> <branch> [<tag>]
|
||||
# <path> is optional but positional, pass "-" to default to the
|
||||
# repo name as the path per git-clone's default
|
||||
function branch_repo {
|
||||
local repo=$1
|
||||
local path=${2:-"-"}
|
||||
local sha=$3
|
||||
local branch=$4
|
||||
local tag=${5:-""}
|
||||
|
||||
local repo_dir
|
||||
if [[ -n $SKIP_PATH || "$path" == "-" ]]; then
|
||||
repo_dir=${repo##*/}
|
||||
else
|
||||
repo_dir=$path
|
||||
fi
|
||||
|
||||
if [[ ! -d $repo_dir ]]; then
|
||||
git clone $repo $repo_dir || true
|
||||
fi
|
||||
|
||||
pushd $repo_dir >/dev/null
|
||||
git fetch origin
|
||||
|
||||
if git branch -r | grep ^origin/${SRC_BRANCH}$; then
|
||||
# Get our local copy of the starting branch up-to-date with the origin
|
||||
git checkout -B $SRC_BRANCH origin/$SRC_BRANCH
|
||||
else
|
||||
# If the source branch is not in the origin just use what we have
|
||||
git checkout $SRC_BRANCH
|
||||
fi
|
||||
|
||||
if ! git branch | grep ${branch}$; then
|
||||
# Create the new branch if it does not exist
|
||||
git branch $branch $sha
|
||||
fi
|
||||
|
||||
if [[ -n $tag ]]; then
|
||||
# tag branch point at $sha
|
||||
git tag -s -m "Branch $branch" -f $tag $sha
|
||||
fi
|
||||
|
||||
# Push the new goodness back up
|
||||
if [[ "$repo" =~ "git.starlingx.io" || "$repo" =~ "opendev.org" ]]; then
|
||||
# Do the Gerrit way
|
||||
|
||||
# set up gerrit remote
|
||||
git review -s
|
||||
|
||||
# push
|
||||
if [[ -z $DRY_RUN ]]; then
|
||||
git push --tags gerrit $branch
|
||||
else
|
||||
echo "### skipping push to $branch"
|
||||
fi
|
||||
|
||||
if [[ "$SRC_BRANCH" != "$BRANCH" ]]; then
|
||||
# Skip .gitreview changes when only tagging
|
||||
update_gitreview $branch
|
||||
fi
|
||||
else
|
||||
# Do the Github way
|
||||
# push
|
||||
if [[ -z $DRY_RUN ]]; then
|
||||
git push --tags -u origin $branch
|
||||
else
|
||||
echo "### skipping push to $branch"
|
||||
fi
|
||||
fi
|
||||
|
||||
popd >/dev/null
|
||||
}
|
||||
|
||||
|
||||
# Read the input
|
||||
while read url path sha x; do
|
||||
# Default to repo name if no path supplied
|
||||
path=${path:-"-"}
|
||||
|
||||
# Default to HEAD if no SHA supplied
|
||||
sha=${sha:-HEAD}
|
||||
|
||||
branch_repo "$url" "$path" "$sha" "$BRANCH" "$TAG"
|
||||
done
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
#!/usr/bin/env python
|
||||
# build-context.py
|
||||
#
|
||||
# Builds a summary of git repos and SHAs for a particular build based on
|
||||
# manifest/default.xml and the build output script CONTEXT.sh
|
||||
#
|
||||
# Loads the CONTEXT.sh script from a StarlingX CenGN build and merge it with
|
||||
# a default.xml manifest to build a YAML file that describes a set of
|
||||
# git repositories and their specific configuration for a build.
|
||||
#
|
||||
# 1. Parse default.xml
|
||||
# - build an object containing the remote+repo with attributes: revision, path
|
||||
# 2. Parse CONTEXT.sh
|
||||
# - Match up the path in the cd command with the path from the manifest, add
|
||||
# the SHA as an attribute
|
||||
# 3. Write output:
|
||||
# <repo-url> <path-from-manifest> <SHA-from-context>
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import urllib
|
||||
|
||||
import xmltodict
|
||||
|
||||
def build_parser():
|
||||
parser = argparse.ArgumentParser(description='build-manifest')
|
||||
parser.add_argument(
|
||||
"manifest",
|
||||
metavar="<name-or-url>",
|
||||
help="Manifest file name or direct URL",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--context",
|
||||
metavar="<name-or-url>",
|
||||
help="Context file name or direct URL",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--remote',
|
||||
metavar="<remote-name>",
|
||||
action='append',
|
||||
dest='remotes',
|
||||
default=[],
|
||||
help='Remote to include (repeat to select multiple remotes)',
|
||||
)
|
||||
return parser
|
||||
|
||||
def load_manifest(name):
|
||||
# Load the manifest XML
|
||||
if "://" in name:
|
||||
# Open a URL
|
||||
fd = urllib.urlopen(name)
|
||||
else:
|
||||
# Open a file
|
||||
fd = open(name)
|
||||
doc = xmltodict.parse(fd.read())
|
||||
fd.close()
|
||||
return doc
|
||||
|
||||
def load_context(name):
|
||||
# Extract the workspace path and git SHA for each repo
|
||||
# (cd ./cgcs-root/stx/stx-config && git checkout -f 22a60625f169202a68b524ac0126afb1d10921cd)\n
|
||||
ctx = {}
|
||||
if "://" in name:
|
||||
# Open a URL
|
||||
fd = urllib.urlopen(name)
|
||||
else:
|
||||
# Open a file
|
||||
fd = open(name)
|
||||
line = fd.readline()
|
||||
while line:
|
||||
s = line.split(' ')
|
||||
# Strip './' from the beginning of the path if it exists
|
||||
path = s[1][2:] if s[1].startswith('./') else s[1]
|
||||
# Strip ')\n' from the end of the SHA if it exists
|
||||
# Don't forget that readline() always ends the line with \n
|
||||
commit = s[6][:-2] if s[6].endswith(')\n') else s[6][:-1]
|
||||
ctx[path] = commit
|
||||
line = fd.readline()
|
||||
fd.close()
|
||||
return ctx
|
||||
|
||||
if __name__ == "__main__":
|
||||
opts = build_parser().parse_args()
|
||||
|
||||
manifest = load_manifest(opts.manifest)
|
||||
|
||||
context = {}
|
||||
if opts.context:
|
||||
context = load_context(opts.context)
|
||||
|
||||
# Map the remotes into a dict
|
||||
remotes = {}
|
||||
for r in manifest['manifest']['remote']:
|
||||
if opts.remotes == [] or r['@name'] in opts.remotes:
|
||||
remotes[r['@name']] = r['@fetch']
|
||||
|
||||
# Get the repos
|
||||
for r in manifest['manifest']['project']:
|
||||
if opts.remotes == [] or r['@remote'] in opts.remotes:
|
||||
commit = ""
|
||||
if r['@path'] in context.keys():
|
||||
# If we have no context this always fails
|
||||
commit = context[r['@path']]
|
||||
print("%s/%s %s %s" % (
|
||||
remotes[r['@remote']],
|
||||
r['@name'],
|
||||
r['@path'],
|
||||
commit,
|
||||
))
|
||||
Loading…
Reference in New Issue