Merge "rewrite release jobs in python"
This commit is contained in:
commit
e4c65cfdf2
|
@ -1,67 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
#
|
|
||||||
# Script to create branches based on changes to the
|
|
||||||
# deliverables files in the openstack/releases repository.
|
|
||||||
#
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
set -x
|
|
||||||
|
|
||||||
TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
|
||||||
source $TOOLSDIR/functions
|
|
||||||
|
|
||||||
function usage {
|
|
||||||
echo "Usage: branch_from_yaml.sh releases_repository [deliverable_files]"
|
|
||||||
echo
|
|
||||||
echo "Example: branch_from_yaml.sh ~/repos/openstack/releases"
|
|
||||||
echo "Example: branch_from_yaml.sh ~/repos/openstack/releases"
|
|
||||||
echo "Example: branch_from_yaml.sh ~/repos/openstack/releases deliverables/mitaka/oslo.config.yaml"
|
|
||||||
}
|
|
||||||
|
|
||||||
if [ $# -lt 1 ]; then
|
|
||||||
echo "ERROR: Please specify releases_repository"
|
|
||||||
echo
|
|
||||||
usage
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
RELEASES_REPO="$1"
|
|
||||||
shift
|
|
||||||
DELIVERABLES="$@"
|
|
||||||
|
|
||||||
setup_temp_space
|
|
||||||
|
|
||||||
echo "Current state of $RELEASES_REPO"
|
|
||||||
(cd $RELEASES_REPO && git show)
|
|
||||||
|
|
||||||
echo "Changed files in the latest commit"
|
|
||||||
# This is the command list_deliverable_branches.py uses to figure out
|
|
||||||
# what files have been touched.
|
|
||||||
(cd $RELEASES_REPO && git diff --name-only --pretty=format: HEAD^)
|
|
||||||
|
|
||||||
change_list_file=$MYTMPDIR/change_list.txt
|
|
||||||
|
|
||||||
echo "Discovered deliverable updates"
|
|
||||||
$TOOLSDIR/list_deliverable_branches.py -r $RELEASES_REPO $DELIVERABLES | tee $change_list_file
|
|
||||||
|
|
||||||
RC=0
|
|
||||||
|
|
||||||
while read repo branch ref; do
|
|
||||||
echo "$repo $branch $ref"
|
|
||||||
$TOOLSDIR/make_branch.sh $repo $branch $ref
|
|
||||||
RC=$(($RC + $?))
|
|
||||||
done < $change_list_file
|
|
||||||
|
|
||||||
exit $RC
|
|
|
@ -0,0 +1,204 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Process all of the release requests in changed files in the commit.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os.path
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
|
PRE_RELEASE_RE = re.compile('''
|
||||||
|
\.(\d+(?:[ab]|rc)+\d*)$
|
||||||
|
''', flags=re.VERBOSE | re.UNICODE)
|
||||||
|
|
||||||
|
BINDIR = os.path.dirname(sys.argv[0])
|
||||||
|
RELEASE_SCRIPT = os.path.join(BINDIR, 'release.sh')
|
||||||
|
BRANCH_SCRIPT = os.path.join(BINDIR, 'make_branch.sh')
|
||||||
|
|
||||||
|
|
||||||
|
def find_modified_deliverable_files(reporoot):
|
||||||
|
"Return a list of files modified by the most recent commit."
|
||||||
|
results = subprocess.check_output(
|
||||||
|
['git', 'diff', '--name-only', '--pretty=format:', 'HEAD^'],
|
||||||
|
cwd=reporoot,
|
||||||
|
).decode('utf-8')
|
||||||
|
filenames = [
|
||||||
|
l.strip()
|
||||||
|
for l in results.splitlines()
|
||||||
|
if l.startswith('deliverables/')
|
||||||
|
]
|
||||||
|
return filenames
|
||||||
|
|
||||||
|
|
||||||
|
def tag_release(repo, series_name, version, diff_start, hash,
|
||||||
|
include_pypi_link, first_full_release, meta_data):
|
||||||
|
try:
|
||||||
|
subprocess.check_call(
|
||||||
|
[RELEASE_SCRIPT, repo, series_name, version,
|
||||||
|
diff_start, hash, include_pypi_link,
|
||||||
|
first_full_release, meta_data]
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
# The error output from the script will be printed to
|
||||||
|
# stderr, so we don't need to do anything else here.
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def make_branch(repo, name, ref):
|
||||||
|
try:
|
||||||
|
subprocess.check_call([BRANCH_SCRIPT, repo, name, ref])
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
# The error output from the script will be
|
||||||
|
# printed to stderr, so we don't need to do
|
||||||
|
# anything else here.
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def process_release_requests(reporoot, filenames, meta_data):
|
||||||
|
"""Return a sequence of tuples containing the new versions.
|
||||||
|
|
||||||
|
Return tuples containing (deliverable name, series name, version
|
||||||
|
number, repository name, hash SHA, include pypi link, first full
|
||||||
|
version)
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Determine which deliverable files to process by taking our
|
||||||
|
# command line arguments or by scanning the git repository
|
||||||
|
# for the most recent change.
|
||||||
|
deliverable_files = filenames
|
||||||
|
if not deliverable_files:
|
||||||
|
deliverable_files = find_modified_deliverable_files(
|
||||||
|
reporoot
|
||||||
|
)
|
||||||
|
|
||||||
|
error_count = 0
|
||||||
|
|
||||||
|
for basename in deliverable_files:
|
||||||
|
filename = os.path.join(reporoot, basename)
|
||||||
|
if not os.path.exists(filename):
|
||||||
|
# The file must have been deleted, skip it.
|
||||||
|
continue
|
||||||
|
with open(filename, 'r', encoding='utf-8') as f:
|
||||||
|
deliverable_data = yaml.load(f.read())
|
||||||
|
|
||||||
|
# If there are no releases listed in this file, skip it.
|
||||||
|
if not deliverable_data.get('releases'):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Map the release version to the release contents so we can
|
||||||
|
# easily get a list of repositories for stable branches.
|
||||||
|
releases_by_version = {
|
||||||
|
r['version']: r
|
||||||
|
for r in deliverable_data.get('releases', [])
|
||||||
|
}
|
||||||
|
|
||||||
|
# Determine whether announcements should include a PyPI
|
||||||
|
# link. Default to no, for service projects, because it is
|
||||||
|
# less irksome to fail to include a link to a thing that
|
||||||
|
# exists than to link to something that does not.
|
||||||
|
include_pypi_link = deliverable_data.get(
|
||||||
|
'include-pypi-link',
|
||||||
|
False,
|
||||||
|
)
|
||||||
|
include_pypi_link = 'yes' if include_pypi_link else 'no'
|
||||||
|
|
||||||
|
# The series name is part of the filename, rather than the file
|
||||||
|
# body. That causes release.sh to be called with series="_independent"
|
||||||
|
# for release:independent projects, and release.sh to use master branch
|
||||||
|
# to evaluate fixed bugs.
|
||||||
|
series_name = os.path.basename(
|
||||||
|
os.path.dirname(os.path.abspath(filename))
|
||||||
|
).lstrip('_')
|
||||||
|
|
||||||
|
all_versions = {
|
||||||
|
rel['version']: rel for rel in deliverable_data['releases']
|
||||||
|
}
|
||||||
|
version = deliverable_data['releases'][-1]['version']
|
||||||
|
this_version = all_versions[version]
|
||||||
|
final_versions = [
|
||||||
|
r['version']
|
||||||
|
for r in deliverable_data['releases']
|
||||||
|
if not PRE_RELEASE_RE.search(r['version'])
|
||||||
|
]
|
||||||
|
first_full_release = 'yes' if (
|
||||||
|
final_versions and
|
||||||
|
this_version['version'] == final_versions[0]
|
||||||
|
) else 'no'
|
||||||
|
diff_start = this_version.get('diff-start', '-')
|
||||||
|
|
||||||
|
# Tag releases.
|
||||||
|
|
||||||
|
for project in this_version['projects']:
|
||||||
|
error_count += tag_release(
|
||||||
|
project['repo'], series_name, version,
|
||||||
|
diff_start, project['hash'], include_pypi_link,
|
||||||
|
first_full_release, meta_data,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create branches.
|
||||||
|
|
||||||
|
for branch in deliverable_data.get('branches', []):
|
||||||
|
location = branch['location']
|
||||||
|
|
||||||
|
if isinstance(location, dict):
|
||||||
|
for repo, sha in sorted(location.items()):
|
||||||
|
error_count += make_branch(repo, branch['name'], sha)
|
||||||
|
|
||||||
|
else:
|
||||||
|
# Assume a single location string that is a valid
|
||||||
|
# reference in the git repository.
|
||||||
|
for proj in releases_by_version[location]['projects']:
|
||||||
|
error_count += make_branch(
|
||||||
|
proj['repo'], branch['name'], branch['location'],
|
||||||
|
)
|
||||||
|
|
||||||
|
return error_count
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
'--meta-data',
|
||||||
|
help='extra metadata to add to the release tag',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'deliverable_file',
|
||||||
|
nargs='*',
|
||||||
|
help='paths to YAML files specifying releases',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--releases-repo', '-r',
|
||||||
|
default='.',
|
||||||
|
help='path to the releases repository for automatic scanning',
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
return process_release_requests(
|
||||||
|
args.releases_repo,
|
||||||
|
args.deliverable_file,
|
||||||
|
args.meta_data,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
|
@ -21,12 +21,12 @@ TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||||
source $TOOLSDIR/functions
|
source $TOOLSDIR/functions
|
||||||
|
|
||||||
function usage {
|
function usage {
|
||||||
echo "Usage: release_from_yaml.sh [(-m|--manual)] releases_repository [deliverable_files]"
|
echo "Usage: process_release_requests.sh [(-m|--manual)] releases_repository [deliverable_files]"
|
||||||
echo " release_from_yaml.sh (-h|--help)"
|
echo " process_release_requests.sh (-h|--help)"
|
||||||
echo
|
echo
|
||||||
echo "Example: release_from_yaml.sh -m ~/repos/openstack/releases"
|
echo "Example: process_release_requests.sh -m ~/repos/openstack/releases"
|
||||||
echo "Example: release_from_yaml.sh ~/repos/openstack/releases"
|
echo "Example: process_release_requests.sh ~/repos/openstack/releases"
|
||||||
echo "Example: release_from_yaml.sh -m ~/repos/openstack/releases deliverables/mitaka/oslo.config.yaml"
|
echo "Example: process_release_requests.sh -m ~/repos/openstack/releases deliverables/mitaka/oslo.config.yaml"
|
||||||
}
|
}
|
||||||
|
|
||||||
OPTS=$(getopt -o hm --long manual,help -n $0 -- "$@")
|
OPTS=$(getopt -o hm --long manual,help -n $0 -- "$@")
|
||||||
|
@ -91,26 +91,9 @@ else
|
||||||
fi
|
fi
|
||||||
RELEASE_META=$(git show --format=full --show-notes=review $parent | egrep -i '(Author|Commit:|Code-Review|Workflow|Change-Id)' | sed -e 's/^ //g' -e 's/^/meta:release:/g')
|
RELEASE_META=$(git show --format=full --show-notes=review $parent | egrep -i '(Author|Commit:|Code-Review|Workflow|Change-Id)' | sed -e 's/^ //g' -e 's/^/meta:release:/g')
|
||||||
|
|
||||||
RC=0
|
$TOOLSDIR/process_release_requests.py \
|
||||||
|
--meta-data "$RELEASE_META" \
|
||||||
|
-r $RELEASES_REPO \
|
||||||
|
$DELIVERABLES
|
||||||
|
|
||||||
echo "Current state of $RELEASES_REPO"
|
exit $?
|
||||||
(cd $RELEASES_REPO && git show)
|
|
||||||
|
|
||||||
echo "Changed files in the latest commit"
|
|
||||||
# This is the command list_deliverable_changes.py uses to figure out
|
|
||||||
# what files have been touched.
|
|
||||||
(cd $RELEASES_REPO && git diff --name-only --pretty=format: HEAD^)
|
|
||||||
|
|
||||||
change_list_file=$MYTMPDIR/change_list.txt
|
|
||||||
|
|
||||||
echo "Discovered deliverable updates"
|
|
||||||
$TOOLSDIR/list_deliverable_changes.py -r $RELEASES_REPO $DELIVERABLES | tee $change_list_file
|
|
||||||
|
|
||||||
echo "Starting tagging"
|
|
||||||
while read deliverable series version diff_start repo hash pypi first_full; do
|
|
||||||
echo "$deliverable $series $version $diff_start $repo $hash $pypi $first_full"
|
|
||||||
$TOOLSDIR/release.sh $repo $series $version $diff_start $hash $pypi $first_full "$RELEASE_META"
|
|
||||||
RC=$(($RC + $?))
|
|
||||||
done < $change_list_file
|
|
||||||
|
|
||||||
exit $RC
|
|
|
@ -13,12 +13,8 @@
|
||||||
# Pass the location of the openstack/releases repo to
|
# Pass the location of the openstack/releases repo to
|
||||||
# release_from_yaml.sh explicitly so it knows where to scan to
|
# release_from_yaml.sh explicitly so it knows where to scan to
|
||||||
# look for modified files.
|
# look for modified files.
|
||||||
~/scripts/release-tools/release_from_yaml.sh $RELEASES_DIR
|
~/scripts/release-tools/process_release_requests.sh $RELEASES_DIR
|
||||||
RC1=$?
|
RC=$?
|
||||||
|
|
||||||
# After we have tagged, create any new branches.
|
exit $RC
|
||||||
~/scripts/release-tools/branch_from_yaml.sh $RELEASES_DIR
|
|
||||||
RC2=$?
|
|
||||||
|
|
||||||
exit $(($RC1 + $RC2))
|
|
||||||
executable: /bin/bash
|
executable: /bin/bash
|
||||||
|
|
Loading…
Reference in New Issue