rewrite release jobs in python
Merge the two release scripts into one python script. Change-Id: Ic9832b58f10b0a8a7f2f11b9200f88913706aa3d Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann
parent
0ef11c16d3
commit
d1f62732ca
@ -1,67 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Script to create branches based on changes to the
|
||||
# deliverables files in the openstack/releases repository.
|
||||
#
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -x
|
||||
|
||||
TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
source $TOOLSDIR/functions
|
||||
|
||||
function usage {
|
||||
echo "Usage: branch_from_yaml.sh releases_repository [deliverable_files]"
|
||||
echo
|
||||
echo "Example: branch_from_yaml.sh ~/repos/openstack/releases"
|
||||
echo "Example: branch_from_yaml.sh ~/repos/openstack/releases"
|
||||
echo "Example: branch_from_yaml.sh ~/repos/openstack/releases deliverables/mitaka/oslo.config.yaml"
|
||||
}
|
||||
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "ERROR: Please specify releases_repository"
|
||||
echo
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RELEASES_REPO="$1"
|
||||
shift
|
||||
DELIVERABLES="$@"
|
||||
|
||||
setup_temp_space
|
||||
|
||||
echo "Current state of $RELEASES_REPO"
|
||||
(cd $RELEASES_REPO && git show)
|
||||
|
||||
echo "Changed files in the latest commit"
|
||||
# This is the command list_deliverable_branches.py uses to figure out
|
||||
# what files have been touched.
|
||||
(cd $RELEASES_REPO && git diff --name-only --pretty=format: HEAD^)
|
||||
|
||||
change_list_file=$MYTMPDIR/change_list.txt
|
||||
|
||||
echo "Discovered deliverable updates"
|
||||
$TOOLSDIR/list_deliverable_branches.py -r $RELEASES_REPO $DELIVERABLES | tee $change_list_file
|
||||
|
||||
RC=0
|
||||
|
||||
while read repo branch ref; do
|
||||
echo "$repo $branch $ref"
|
||||
$TOOLSDIR/make_branch.sh $repo $branch $ref
|
||||
RC=$(($RC + $?))
|
||||
done < $change_list_file
|
||||
|
||||
exit $RC
|
||||
204
jenkins/scripts/release-tools/process_release_requests.py
Executable file
204
jenkins/scripts/release-tools/process_release_requests.py
Executable file
@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Process all of the release requests in changed files in the commit.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os.path
|
||||
import re
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
PRE_RELEASE_RE = re.compile('''
|
||||
\.(\d+(?:[ab]|rc)+\d*)$
|
||||
''', flags=re.VERBOSE | re.UNICODE)
|
||||
|
||||
BINDIR = os.path.dirname(sys.argv[0])
|
||||
RELEASE_SCRIPT = os.path.join(BINDIR, 'release.sh')
|
||||
BRANCH_SCRIPT = os.path.join(BINDIR, 'make_branch.sh')
|
||||
|
||||
|
||||
def find_modified_deliverable_files(reporoot):
|
||||
"Return a list of files modified by the most recent commit."
|
||||
results = subprocess.check_output(
|
||||
['git', 'diff', '--name-only', '--pretty=format:', 'HEAD^'],
|
||||
cwd=reporoot,
|
||||
).decode('utf-8')
|
||||
filenames = [
|
||||
l.strip()
|
||||
for l in results.splitlines()
|
||||
if l.startswith('deliverables/')
|
||||
]
|
||||
return filenames
|
||||
|
||||
|
||||
def tag_release(repo, series_name, version, diff_start, hash,
|
||||
include_pypi_link, first_full_release, meta_data):
|
||||
try:
|
||||
subprocess.check_call(
|
||||
[RELEASE_SCRIPT, repo, series_name, version,
|
||||
diff_start, hash, include_pypi_link,
|
||||
first_full_release, meta_data]
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
# The error output from the script will be printed to
|
||||
# stderr, so we don't need to do anything else here.
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def make_branch(repo, name, ref):
|
||||
try:
|
||||
subprocess.check_call([BRANCH_SCRIPT, repo, name, ref])
|
||||
except subprocess.CalledProcessError:
|
||||
# The error output from the script will be
|
||||
# printed to stderr, so we don't need to do
|
||||
# anything else here.
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def process_release_requests(reporoot, filenames, meta_data):
|
||||
"""Return a sequence of tuples containing the new versions.
|
||||
|
||||
Return tuples containing (deliverable name, series name, version
|
||||
number, repository name, hash SHA, include pypi link, first full
|
||||
version)
|
||||
|
||||
"""
|
||||
|
||||
# Determine which deliverable files to process by taking our
|
||||
# command line arguments or by scanning the git repository
|
||||
# for the most recent change.
|
||||
deliverable_files = filenames
|
||||
if not deliverable_files:
|
||||
deliverable_files = find_modified_deliverable_files(
|
||||
reporoot
|
||||
)
|
||||
|
||||
error_count = 0
|
||||
|
||||
for basename in deliverable_files:
|
||||
filename = os.path.join(reporoot, basename)
|
||||
if not os.path.exists(filename):
|
||||
# The file must have been deleted, skip it.
|
||||
continue
|
||||
with open(filename, 'r', encoding='utf-8') as f:
|
||||
deliverable_data = yaml.load(f.read())
|
||||
|
||||
# If there are no releases listed in this file, skip it.
|
||||
if not deliverable_data.get('releases'):
|
||||
continue
|
||||
|
||||
# Map the release version to the release contents so we can
|
||||
# easily get a list of repositories for stable branches.
|
||||
releases_by_version = {
|
||||
r['version']: r
|
||||
for r in deliverable_data.get('releases', [])
|
||||
}
|
||||
|
||||
# Determine whether announcements should include a PyPI
|
||||
# link. Default to no, for service projects, because it is
|
||||
# less irksome to fail to include a link to a thing that
|
||||
# exists than to link to something that does not.
|
||||
include_pypi_link = deliverable_data.get(
|
||||
'include-pypi-link',
|
||||
False,
|
||||
)
|
||||
include_pypi_link = 'yes' if include_pypi_link else 'no'
|
||||
|
||||
# The series name is part of the filename, rather than the file
|
||||
# body. That causes release.sh to be called with series="_independent"
|
||||
# for release:independent projects, and release.sh to use master branch
|
||||
# to evaluate fixed bugs.
|
||||
series_name = os.path.basename(
|
||||
os.path.dirname(os.path.abspath(filename))
|
||||
).lstrip('_')
|
||||
|
||||
all_versions = {
|
||||
rel['version']: rel for rel in deliverable_data['releases']
|
||||
}
|
||||
version = deliverable_data['releases'][-1]['version']
|
||||
this_version = all_versions[version]
|
||||
final_versions = [
|
||||
r['version']
|
||||
for r in deliverable_data['releases']
|
||||
if not PRE_RELEASE_RE.search(r['version'])
|
||||
]
|
||||
first_full_release = 'yes' if (
|
||||
final_versions and
|
||||
this_version['version'] == final_versions[0]
|
||||
) else 'no'
|
||||
diff_start = this_version.get('diff-start', '-')
|
||||
|
||||
# Tag releases.
|
||||
|
||||
for project in this_version['projects']:
|
||||
error_count += tag_release(
|
||||
project['repo'], series_name, version,
|
||||
diff_start, project['hash'], include_pypi_link,
|
||||
first_full_release, meta_data,
|
||||
)
|
||||
|
||||
# Create branches.
|
||||
|
||||
for branch in deliverable_data.get('branches', []):
|
||||
location = branch['location']
|
||||
|
||||
if isinstance(location, dict):
|
||||
for repo, sha in sorted(location.items()):
|
||||
error_count += make_branch(repo, branch['name'], sha)
|
||||
|
||||
else:
|
||||
# Assume a single location string that is a valid
|
||||
# reference in the git repository.
|
||||
for proj in releases_by_version[location]['projects']:
|
||||
error_count += make_branch(
|
||||
proj['repo'], branch['name'], branch['location'],
|
||||
)
|
||||
|
||||
return error_count
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'--meta-data',
|
||||
help='extra metadata to add to the release tag',
|
||||
)
|
||||
parser.add_argument(
|
||||
'deliverable_file',
|
||||
nargs='*',
|
||||
help='paths to YAML files specifying releases',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--releases-repo', '-r',
|
||||
default='.',
|
||||
help='path to the releases repository for automatic scanning',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
return process_release_requests(
|
||||
args.releases_repo,
|
||||
args.deliverable_file,
|
||||
args.meta_data,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -21,12 +21,12 @@ TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
source $TOOLSDIR/functions
|
||||
|
||||
function usage {
|
||||
echo "Usage: release_from_yaml.sh [(-m|--manual)] releases_repository [deliverable_files]"
|
||||
echo " release_from_yaml.sh (-h|--help)"
|
||||
echo "Usage: process_release_requests.sh [(-m|--manual)] releases_repository [deliverable_files]"
|
||||
echo " process_release_requests.sh (-h|--help)"
|
||||
echo
|
||||
echo "Example: release_from_yaml.sh -m ~/repos/openstack/releases"
|
||||
echo "Example: release_from_yaml.sh ~/repos/openstack/releases"
|
||||
echo "Example: release_from_yaml.sh -m ~/repos/openstack/releases deliverables/mitaka/oslo.config.yaml"
|
||||
echo "Example: process_release_requests.sh -m ~/repos/openstack/releases"
|
||||
echo "Example: process_release_requests.sh ~/repos/openstack/releases"
|
||||
echo "Example: process_release_requests.sh -m ~/repos/openstack/releases deliverables/mitaka/oslo.config.yaml"
|
||||
}
|
||||
|
||||
OPTS=$(getopt -o hm --long manual,help -n $0 -- "$@")
|
||||
@ -91,26 +91,9 @@ else
|
||||
fi
|
||||
RELEASE_META=$(git show --format=full --show-notes=review $parent | egrep -i '(Author|Commit:|Code-Review|Workflow|Change-Id)' | sed -e 's/^ //g' -e 's/^/meta:release:/g')
|
||||
|
||||
RC=0
|
||||
$TOOLSDIR/process_release_requests.py \
|
||||
--meta-data "$RELEASE_META" \
|
||||
-r $RELEASES_REPO \
|
||||
$DELIVERABLES
|
||||
|
||||
echo "Current state of $RELEASES_REPO"
|
||||
(cd $RELEASES_REPO && git show)
|
||||
|
||||
echo "Changed files in the latest commit"
|
||||
# This is the command list_deliverable_changes.py uses to figure out
|
||||
# what files have been touched.
|
||||
(cd $RELEASES_REPO && git diff --name-only --pretty=format: HEAD^)
|
||||
|
||||
change_list_file=$MYTMPDIR/change_list.txt
|
||||
|
||||
echo "Discovered deliverable updates"
|
||||
$TOOLSDIR/list_deliverable_changes.py -r $RELEASES_REPO $DELIVERABLES | tee $change_list_file
|
||||
|
||||
echo "Starting tagging"
|
||||
while read deliverable series version diff_start repo hash pypi first_full; do
|
||||
echo "$deliverable $series $version $diff_start $repo $hash $pypi $first_full"
|
||||
$TOOLSDIR/release.sh $repo $series $version $diff_start $hash $pypi $first_full "$RELEASE_META"
|
||||
RC=$(($RC + $?))
|
||||
done < $change_list_file
|
||||
|
||||
exit $RC
|
||||
exit $?
|
||||
@ -13,12 +13,8 @@
|
||||
# Pass the location of the openstack/releases repo to
|
||||
# release_from_yaml.sh explicitly so it knows where to scan to
|
||||
# look for modified files.
|
||||
~/scripts/release-tools/release_from_yaml.sh $RELEASES_DIR
|
||||
RC1=$?
|
||||
~/scripts/release-tools/process_release_requests.sh $RELEASES_DIR
|
||||
RC=$?
|
||||
|
||||
# After we have tagged, create any new branches.
|
||||
~/scripts/release-tools/branch_from_yaml.sh $RELEASES_DIR
|
||||
RC2=$?
|
||||
|
||||
exit $(($RC1 + $RC2))
|
||||
exit $RC
|
||||
executable: /bin/bash
|
||||
|
||||
Loading…
Reference in New Issue
Block a user