From 8dc7f193ef6ce3a3ae75e2ff506e6fe0a21de0f2 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Sat, 3 Aug 2019 12:48:56 -0500 Subject: [PATCH] Add release scripts build-context.py and branch-repo.sh These scripts replace the earlier branch-stx.sh and getrepo.sh and are more generally useful in that the assumptions regarding branch and tag naming have all been removed. build-context.py pulls repo information from a manifest file and optionally adds the specific context SHAs from the build output script CONTEXT.sh. By default it processes all repos from the all remotes in the manifest. Use --remote to select one or more remotes (repeat option for more than one remote). The result can be fed into branch-repo.sh to create new branches and/or tags corresponding to a specific build context. For example, to fetch only the starlingx remote repos: release/build-context.py \ --remote starlingx \ --context http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/latest_green_build/outputs/CONTEXT.sh \ https://opendev.org/starlingx/manifest/raw/branch/master/default.xml | \ release/branch-repo.sh -b r/stx.2.0 -t v2.0.0.rc0 branch-repo.sh makes a number of small assumption changes from branch-stx.sh in the branch_repo() function. It corrects a stray merge commit being introduced when attempting to only tag a branch that has not been pushed to the origin yet. This might occur during a series of local tests using --dry-run. It also allows passing in a path value rather than just assuming the repo name. Change-Id: I3d6a510cc2c37d10c468ac273178b3bad8e42c01 Signed-off-by: Dean Troyer --- release/README.rst | 37 +++++++ release/branch-repo.sh | 205 +++++++++++++++++++++++++++++++++++++++ release/build-context.py | 109 +++++++++++++++++++++ 3 files changed, 351 insertions(+) create mode 100644 release/README.rst create mode 100755 release/branch-repo.sh create mode 100755 release/build-context.py diff --git a/release/README.rst b/release/README.rst new file mode 100644 index 00000000..490c5df1 --- /dev/null +++ b/release/README.rst @@ -0,0 +1,37 @@ +============= +Release Tools +============= + +A set of tools used in the StarlingX release process past, current and future... + +``branch-repo.sh`` - Derived from the older ``branch-stx.sh`` to support +reading a list of repos, paths and SHAs from stdin, usually the output of +``build-context.py``. This simplifies the creation of branching based on +a specific build using the CONTEXT.sh file generated by the build system. + +``build-context.py`` - Takes a manifest file and an optional CONTEXT.sh file +to produce a list of repositories and their paths and SHAs that represent a +specific build. This can be fed directly into ``branch-repo.sh`` to create +branches corresponding to that specific build. (This is in Python completely +due to combining XML and the joys of dict-like manipulation in shell.) + +Examples +-------- + +Here is an example of creating a new branch named 'r/stx.2.0' and tag 'v2.0.0.rc0' from +the lastest green build context at the time: + +:: + + release/build-context.py \ + --remote starlingx \ + --context http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/latest_green_build/outputs/CONTEXT.sh \ + https://opendev.org/starlingx/manifest/raw/branch/master/default.xml | \ + release/branch-repo.sh -b r/stx.2.0 -t v2.0.0.rc0 + +Older Scripts +============= + +branch-stx.sh - Shell script previously used to create release and milestone branches. This has a number of assumptions baked in regarding branch and tag names. + +get-repo.sh - Called from ``branch-stx.sh`` to read the XML manifest and extract repo names from specific remotes. This has totally been absorbed into ``build-context.py``. diff --git a/release/branch-repo.sh b/release/branch-repo.sh new file mode 100755 index 00000000..59f47711 --- /dev/null +++ b/release/branch-repo.sh @@ -0,0 +1,205 @@ +#!/bin/bash +# branch-repo.sh - Create new branches in a set of Git repositories +# +# branch-repo.sh [--dry-run|-n] [-b ] [-t ] [-s ] [-i] +# +# --dry-run|-n Do all work except pushing back to the remote repo. +# Useful to validate everything locally before pushing. +# +# -b Name of the new branch +# +# -t Apply a tag at the SHA passed in the input data, or HEAD +# of the new branch if no SHA is present +# +# -s The starting branch to use instead of the default 'master'. +# This is needed when the working branch is not named 'master'. +# Setting == makes this a tag-only +# operation (.gitreview updates are skipped). +# +# -i Ignore path in input; use the last component of the repo +# name for the path similar to git clone's default. +# +# Read a list of repo tuples from stdin: +# +# +# For each repo: +# * create a new branch at , or at HEAD of SRC_BRANCH if no +# * tag the new branch with an initial release identifier if is set +# * update the .gitreview file to default to the new branch (Gerrit repos only) +# +# Some environment variables are available for modifying this script's behaviour +# NOTE: The command-line options override the environment variables when +# both are present. +# +# - BRANCH sets the new branch name +# +# - SRC_BRANCH sets the source branch name . +# +# - TAG sets the release tag . +# +# More Notes +# * The detection to use Gerrit or Github is determined by the presence of +# 'git.starlingx.io' or 'opendev.org' in the repo URL. This may be +# sub-optimal. The only actual difference in execution is .gitreview +# updates are only prepared for Gerrit repos. + +set -e + +# Defaults +BRANCH=${BRANCH:-""} +SRC_BRANCH=${SRC_BRANCH:-master} +TAG=${TAG:-""} + +optspec="b:ins:t:-:" +while getopts "$optspec" o; do + case "${o}" in + # Hack in longopt support + -) + case "${OPTARG}" in + dry-run) + DRY_RUN=1 + ;; + *) + if [[ "$OPTERR" = 1 ]] && [[ "${optspec:0:1}" != ":" ]]; then + echo "Unknown option --${OPTARG}" >&2 + fi + ;; + + esac + ;; + b) + BRANCH=${OPTARG} + ;; + i) + SKIP_PATH=1 + ;; + n) + DRY_RUN=1 + ;; + s) + SRC_BRANCH=${OPTARG} + ;; + t) + TAG=${OPTARG} + ;; + esac +done +shift $((OPTIND-1)) + +# See if we can build a repo list +if [[ -z $BRANCH ]]; then + echo "ERROR: No repos to process" + echo "Usage: $0 [--dry-run|-n] [-b ] [-t ] [-s ] [-i]" + exit 1 +fi + +# This is where other scripts live that we need +script_dir=$(realpath $(dirname $0)) + +# update_gitreview +# Based on update_gitreview() from https://github.com/openstack/releases/blob/a7db6cf156ba66d50e1955db2163506365182ee8/tools/functions#L67 +function update_gitreview { + typeset branch="$1" + + git checkout $branch + # Remove a trailing newline, if present, to ensure consistent + # formatting when we add the defaultbranch line next. + typeset grcontents="$(echo -n "$(cat .gitreview | grep -v defaultbranch)") +defaultbranch=$branch" + echo "$grcontents" > .gitreview + git add .gitreview + if git commit -s -m "Update .gitreview for $branch"; then + if [[ -z $DRY_RUN ]]; then + git review -t "create-${branch}" + else + echo "### skipping .gitreview submission to $branch" + fi + else + echo "### no changes required for .gitreview" + fi +} + +# branch_repo [] +# is optional but positional, pass "-" to default to the +# repo name as the path per git-clone's default +function branch_repo { + local repo=$1 + local path=${2:-"-"} + local sha=$3 + local branch=$4 + local tag=${5:-""} + + local repo_dir + if [[ -n $SKIP_PATH || "$path" == "-" ]]; then + repo_dir=${repo##*/} + else + repo_dir=$path + fi + + if [[ ! -d $repo_dir ]]; then + git clone $repo $repo_dir || true + fi + + pushd $repo_dir >/dev/null + git fetch origin + + if git branch -r | grep ^origin/${SRC_BRANCH}$; then + # Get our local copy of the starting branch up-to-date with the origin + git checkout -B $SRC_BRANCH origin/$SRC_BRANCH + else + # If the source branch is not in the origin just use what we have + git checkout $SRC_BRANCH + fi + + if ! git branch | grep ${branch}$; then + # Create the new branch if it does not exist + git branch $branch $sha + fi + + if [[ -n $tag ]]; then + # tag branch point at $sha + git tag -s -m "Branch $branch" -f $tag $sha + fi + + # Push the new goodness back up + if [[ "$repo" =~ "git.starlingx.io" || "$repo" =~ "opendev.org" ]]; then + # Do the Gerrit way + + # set up gerrit remote + git review -s + + # push + if [[ -z $DRY_RUN ]]; then + git push --tags gerrit $branch + else + echo "### skipping push to $branch" + fi + + if [[ "$SRC_BRANCH" != "$BRANCH" ]]; then + # Skip .gitreview changes when only tagging + update_gitreview $branch + fi + else + # Do the Github way + # push + if [[ -z $DRY_RUN ]]; then + git push --tags -u origin $branch + else + echo "### skipping push to $branch" + fi + fi + + popd >/dev/null +} + + +# Read the input +while read url path sha x; do + # Default to repo name if no path supplied + path=${path:-"-"} + + # Default to HEAD if no SHA supplied + sha=${sha:-HEAD} + + branch_repo "$url" "$path" "$sha" "$BRANCH" "$TAG" +done diff --git a/release/build-context.py b/release/build-context.py new file mode 100755 index 00000000..a5c9b382 --- /dev/null +++ b/release/build-context.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# build-context.py +# +# Builds a summary of git repos and SHAs for a particular build based on +# manifest/default.xml and the build output script CONTEXT.sh +# +# Loads the CONTEXT.sh script from a StarlingX CenGN build and merge it with +# a default.xml manifest to build a YAML file that describes a set of +# git repositories and their specific configuration for a build. +# +# 1. Parse default.xml +# - build an object containing the remote+repo with attributes: revision, path +# 2. Parse CONTEXT.sh +# - Match up the path in the cd command with the path from the manifest, add +# the SHA as an attribute +# 3. Write output: +# + +import argparse +import sys +import urllib + +import xmltodict + +def build_parser(): + parser = argparse.ArgumentParser(description='build-manifest') + parser.add_argument( + "manifest", + metavar="", + help="Manifest file name or direct URL", + ) + parser.add_argument( + "--context", + metavar="", + help="Context file name or direct URL", + ) + parser.add_argument( + '--remote', + metavar="", + action='append', + dest='remotes', + default=[], + help='Remote to include (repeat to select multiple remotes)', + ) + return parser + +def load_manifest(name): + # Load the manifest XML + if "://" in name: + # Open a URL + fd = urllib.urlopen(name) + else: + # Open a file + fd = open(name) + doc = xmltodict.parse(fd.read()) + fd.close() + return doc + +def load_context(name): + # Extract the workspace path and git SHA for each repo + # (cd ./cgcs-root/stx/stx-config && git checkout -f 22a60625f169202a68b524ac0126afb1d10921cd)\n + ctx = {} + if "://" in name: + # Open a URL + fd = urllib.urlopen(name) + else: + # Open a file + fd = open(name) + line = fd.readline() + while line: + s = line.split(' ') + # Strip './' from the beginning of the path if it exists + path = s[1][2:] if s[1].startswith('./') else s[1] + # Strip ')\n' from the end of the SHA if it exists + # Don't forget that readline() always ends the line with \n + commit = s[6][:-2] if s[6].endswith(')\n') else s[6][:-1] + ctx[path] = commit + line = fd.readline() + fd.close() + return ctx + +if __name__ == "__main__": + opts = build_parser().parse_args() + + manifest = load_manifest(opts.manifest) + + context = {} + if opts.context: + context = load_context(opts.context) + + # Map the remotes into a dict + remotes = {} + for r in manifest['manifest']['remote']: + if opts.remotes == [] or r['@name'] in opts.remotes: + remotes[r['@name']] = r['@fetch'] + + # Get the repos + for r in manifest['manifest']['project']: + if opts.remotes == [] or r['@remote'] in opts.remotes: + commit = "" + if r['@path'] in context.keys(): + # If we have no context this always fails + commit = context[r['@path']] + print("%s/%s %s %s" % ( + remotes[r['@remote']], + r['@name'], + r['@path'], + commit, + ))