From d1f62732ca042c0bc4eac1e7068063366c68895c Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 24 Oct 2017 14:23:57 -0400 Subject: [PATCH] rewrite release jobs in python Merge the two release scripts into one python script. Change-Id: Ic9832b58f10b0a8a7f2f11b9200f88913706aa3d Signed-off-by: Doug Hellmann --- .../scripts/release-tools/branch_from_yaml.sh | 67 ------ .../release-tools/process_release_requests.py | 204 ++++++++++++++++++ ...om_yaml.sh => process_release_requests.sh} | 37 +--- playbooks/release/tag.yaml | 10 +- 4 files changed, 217 insertions(+), 101 deletions(-) delete mode 100755 jenkins/scripts/release-tools/branch_from_yaml.sh create mode 100755 jenkins/scripts/release-tools/process_release_requests.py rename jenkins/scripts/release-tools/{release_from_yaml.sh => process_release_requests.sh} (66%) diff --git a/jenkins/scripts/release-tools/branch_from_yaml.sh b/jenkins/scripts/release-tools/branch_from_yaml.sh deleted file mode 100755 index e8e17d54ae..0000000000 --- a/jenkins/scripts/release-tools/branch_from_yaml.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# -# Script to create branches based on changes to the -# deliverables files in the openstack/releases repository. -# -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -x - -TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source $TOOLSDIR/functions - -function usage { - echo "Usage: branch_from_yaml.sh releases_repository [deliverable_files]" - echo - echo "Example: branch_from_yaml.sh ~/repos/openstack/releases" - echo "Example: branch_from_yaml.sh ~/repos/openstack/releases" - echo "Example: branch_from_yaml.sh ~/repos/openstack/releases deliverables/mitaka/oslo.config.yaml" -} - -if [ $# -lt 1 ]; then - echo "ERROR: Please specify releases_repository" - echo - usage - exit 1 -fi - -RELEASES_REPO="$1" -shift -DELIVERABLES="$@" - -setup_temp_space - -echo "Current state of $RELEASES_REPO" -(cd $RELEASES_REPO && git show) - -echo "Changed files in the latest commit" -# This is the command list_deliverable_branches.py uses to figure out -# what files have been touched. -(cd $RELEASES_REPO && git diff --name-only --pretty=format: HEAD^) - -change_list_file=$MYTMPDIR/change_list.txt - -echo "Discovered deliverable updates" -$TOOLSDIR/list_deliverable_branches.py -r $RELEASES_REPO $DELIVERABLES | tee $change_list_file - -RC=0 - -while read repo branch ref; do - echo "$repo $branch $ref" - $TOOLSDIR/make_branch.sh $repo $branch $ref - RC=$(($RC + $?)) -done < $change_list_file - -exit $RC diff --git a/jenkins/scripts/release-tools/process_release_requests.py b/jenkins/scripts/release-tools/process_release_requests.py new file mode 100755 index 0000000000..caf61bd556 --- /dev/null +++ b/jenkins/scripts/release-tools/process_release_requests.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Process all of the release requests in changed files in the commit. +""" + +import argparse +import os.path +import re +import sys +import subprocess + +import yaml + + +PRE_RELEASE_RE = re.compile(''' + \.(\d+(?:[ab]|rc)+\d*)$ +''', flags=re.VERBOSE | re.UNICODE) + +BINDIR = os.path.dirname(sys.argv[0]) +RELEASE_SCRIPT = os.path.join(BINDIR, 'release.sh') +BRANCH_SCRIPT = os.path.join(BINDIR, 'make_branch.sh') + + +def find_modified_deliverable_files(reporoot): + "Return a list of files modified by the most recent commit." + results = subprocess.check_output( + ['git', 'diff', '--name-only', '--pretty=format:', 'HEAD^'], + cwd=reporoot, + ).decode('utf-8') + filenames = [ + l.strip() + for l in results.splitlines() + if l.startswith('deliverables/') + ] + return filenames + + +def tag_release(repo, series_name, version, diff_start, hash, + include_pypi_link, first_full_release, meta_data): + try: + subprocess.check_call( + [RELEASE_SCRIPT, repo, series_name, version, + diff_start, hash, include_pypi_link, + first_full_release, meta_data] + ) + except subprocess.CalledProcessError: + # The error output from the script will be printed to + # stderr, so we don't need to do anything else here. + return 1 + return 0 + + +def make_branch(repo, name, ref): + try: + subprocess.check_call([BRANCH_SCRIPT, repo, name, ref]) + except subprocess.CalledProcessError: + # The error output from the script will be + # printed to stderr, so we don't need to do + # anything else here. + return 1 + return 0 + + +def process_release_requests(reporoot, filenames, meta_data): + """Return a sequence of tuples containing the new versions. + + Return tuples containing (deliverable name, series name, version + number, repository name, hash SHA, include pypi link, first full + version) + + """ + + # Determine which deliverable files to process by taking our + # command line arguments or by scanning the git repository + # for the most recent change. + deliverable_files = filenames + if not deliverable_files: + deliverable_files = find_modified_deliverable_files( + reporoot + ) + + error_count = 0 + + for basename in deliverable_files: + filename = os.path.join(reporoot, basename) + if not os.path.exists(filename): + # The file must have been deleted, skip it. + continue + with open(filename, 'r', encoding='utf-8') as f: + deliverable_data = yaml.load(f.read()) + + # If there are no releases listed in this file, skip it. + if not deliverable_data.get('releases'): + continue + + # Map the release version to the release contents so we can + # easily get a list of repositories for stable branches. + releases_by_version = { + r['version']: r + for r in deliverable_data.get('releases', []) + } + + # Determine whether announcements should include a PyPI + # link. Default to no, for service projects, because it is + # less irksome to fail to include a link to a thing that + # exists than to link to something that does not. + include_pypi_link = deliverable_data.get( + 'include-pypi-link', + False, + ) + include_pypi_link = 'yes' if include_pypi_link else 'no' + + # The series name is part of the filename, rather than the file + # body. That causes release.sh to be called with series="_independent" + # for release:independent projects, and release.sh to use master branch + # to evaluate fixed bugs. + series_name = os.path.basename( + os.path.dirname(os.path.abspath(filename)) + ).lstrip('_') + + all_versions = { + rel['version']: rel for rel in deliverable_data['releases'] + } + version = deliverable_data['releases'][-1]['version'] + this_version = all_versions[version] + final_versions = [ + r['version'] + for r in deliverable_data['releases'] + if not PRE_RELEASE_RE.search(r['version']) + ] + first_full_release = 'yes' if ( + final_versions and + this_version['version'] == final_versions[0] + ) else 'no' + diff_start = this_version.get('diff-start', '-') + + # Tag releases. + + for project in this_version['projects']: + error_count += tag_release( + project['repo'], series_name, version, + diff_start, project['hash'], include_pypi_link, + first_full_release, meta_data, + ) + + # Create branches. + + for branch in deliverable_data.get('branches', []): + location = branch['location'] + + if isinstance(location, dict): + for repo, sha in sorted(location.items()): + error_count += make_branch(repo, branch['name'], sha) + + else: + # Assume a single location string that is a valid + # reference in the git repository. + for proj in releases_by_version[location]['projects']: + error_count += make_branch( + proj['repo'], branch['name'], branch['location'], + ) + + return error_count + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--meta-data', + help='extra metadata to add to the release tag', + ) + parser.add_argument( + 'deliverable_file', + nargs='*', + help='paths to YAML files specifying releases', + ) + parser.add_argument( + '--releases-repo', '-r', + default='.', + help='path to the releases repository for automatic scanning', + ) + args = parser.parse_args() + + return process_release_requests( + args.releases_repo, + args.deliverable_file, + args.meta_data, + ) + + +if __name__ == '__main__': + main() diff --git a/jenkins/scripts/release-tools/release_from_yaml.sh b/jenkins/scripts/release-tools/process_release_requests.sh similarity index 66% rename from jenkins/scripts/release-tools/release_from_yaml.sh rename to jenkins/scripts/release-tools/process_release_requests.sh index 4debbd58d8..9f201a4032 100755 --- a/jenkins/scripts/release-tools/release_from_yaml.sh +++ b/jenkins/scripts/release-tools/process_release_requests.sh @@ -21,12 +21,12 @@ TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $TOOLSDIR/functions function usage { - echo "Usage: release_from_yaml.sh [(-m|--manual)] releases_repository [deliverable_files]" - echo " release_from_yaml.sh (-h|--help)" + echo "Usage: process_release_requests.sh [(-m|--manual)] releases_repository [deliverable_files]" + echo " process_release_requests.sh (-h|--help)" echo - echo "Example: release_from_yaml.sh -m ~/repos/openstack/releases" - echo "Example: release_from_yaml.sh ~/repos/openstack/releases" - echo "Example: release_from_yaml.sh -m ~/repos/openstack/releases deliverables/mitaka/oslo.config.yaml" + echo "Example: process_release_requests.sh -m ~/repos/openstack/releases" + echo "Example: process_release_requests.sh ~/repos/openstack/releases" + echo "Example: process_release_requests.sh -m ~/repos/openstack/releases deliverables/mitaka/oslo.config.yaml" } OPTS=$(getopt -o hm --long manual,help -n $0 -- "$@") @@ -91,26 +91,9 @@ else fi RELEASE_META=$(git show --format=full --show-notes=review $parent | egrep -i '(Author|Commit:|Code-Review|Workflow|Change-Id)' | sed -e 's/^ //g' -e 's/^/meta:release:/g') -RC=0 +$TOOLSDIR/process_release_requests.py \ + --meta-data "$RELEASE_META" \ + -r $RELEASES_REPO \ + $DELIVERABLES -echo "Current state of $RELEASES_REPO" -(cd $RELEASES_REPO && git show) - -echo "Changed files in the latest commit" -# This is the command list_deliverable_changes.py uses to figure out -# what files have been touched. -(cd $RELEASES_REPO && git diff --name-only --pretty=format: HEAD^) - -change_list_file=$MYTMPDIR/change_list.txt - -echo "Discovered deliverable updates" -$TOOLSDIR/list_deliverable_changes.py -r $RELEASES_REPO $DELIVERABLES | tee $change_list_file - -echo "Starting tagging" -while read deliverable series version diff_start repo hash pypi first_full; do - echo "$deliverable $series $version $diff_start $repo $hash $pypi $first_full" - $TOOLSDIR/release.sh $repo $series $version $diff_start $hash $pypi $first_full "$RELEASE_META" - RC=$(($RC + $?)) -done < $change_list_file - -exit $RC +exit $? diff --git a/playbooks/release/tag.yaml b/playbooks/release/tag.yaml index 72f127ef6d..b8cb8b3bbc 100644 --- a/playbooks/release/tag.yaml +++ b/playbooks/release/tag.yaml @@ -13,12 +13,8 @@ # Pass the location of the openstack/releases repo to # release_from_yaml.sh explicitly so it knows where to scan to # look for modified files. - ~/scripts/release-tools/release_from_yaml.sh $RELEASES_DIR - RC1=$? + ~/scripts/release-tools/process_release_requests.sh $RELEASES_DIR + RC=$? - # After we have tagged, create any new branches. - ~/scripts/release-tools/branch_from_yaml.sh $RELEASES_DIR - RC2=$? - - exit $(($RC1 + $RC2)) + exit $RC executable: /bin/bash