import the release tools that need to run on secure nodes

Move the scripts that need to run on secure nodes to automatically tag
releases from release-tools to this repo.

Change-Id: I66e15cd0fa270e9b7a5cb3849cb8d14fa4c617b1
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann 2016-06-21 15:31:31 -04:00
parent e5a508a5de
commit a31c54473c
7 changed files with 581 additions and 0 deletions

View File

@ -0,0 +1,52 @@
#!/bin/bash
#
# Script to create stable branches based on changes to the
# deliverables files in the openstack/releases repository.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -e
TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $TOOLSDIR/functions
function usage {
echo "Usage: branch_from_yaml.sh releases_repository series [deliverable_files]"
echo
echo "Example: branch_from_yaml.sh ~/repos/openstack/releases mitaka"
echo "Example: branch_from_yaml.sh ~/repos/openstack/releases mitaka"
echo "Example: branch_from_yaml.sh ~/repos/openstack/releases mitaka deliverables/mitaka/oslo.config.yaml"
}
if [ $# -lt 2 ]; then
echo "ERROR: Please specify releases_repository and series"
echo
usage
exit 1
fi
RELEASES_REPO="$1"
shift
SERIES="$1"
shift
DELIVERABLES="$@"
$TOOLSDIR/list_deliverable_changes.py -r $RELEASES_REPO $DELIVERABLES \
| while read deliverable ignore_series version repo ignore_hash ignore_announce_to pypi ignore_first_full; do
title "$repo $series $version"
$TOOLSDIR/make_stable_branch.sh $SERIES $repo $version
done
exit 0

View File

@ -0,0 +1,100 @@
#!/bin/bash
#
# Shared functions for shell scripts
#
# Make sure custom grep options don't get in the way
unset GREP_OPTIONS
function lp_project_to_repo {
typeset proj="$1"
if [[ $proj == python-*client* ]]; then
echo $proj
elif [[ $proj == glance-store ]]; then
echo glance_store
elif [[ $proj == django-openstack-auth ]]; then
echo django_openstack_auth
else
# Some of the repository names don't match the launchpad names, e.g.
# python-stevedore and python-cliff.
echo $proj | sed -e 's|^python-||'
fi
}
function title {
echo
if [ -t 1 ]; then
echo "$(tput bold)$(tput setaf 1)[ $1 ]$(tput sgr0)"
else
echo "[ $1 ]"
fi
}
function _cleanup_tmp {
rm -rf $MYTMPDIR
return 0
}
function setup_temp_space {
MYTMPDIR=`mktemp -d _tmp-${1}-XXX`
mkdir -p "$MYTMPDIR"
trap _cleanup_tmp EXIT
cd "$MYTMPDIR"
# NOTE(dhellmann): On some platforms mktemp returns a short name
# instead of a full path, so expand the full path by looking at
# where we ended up after the cd operation.
MYTMPDIR="$(pwd)"
}
function get_last_tag {
# Print the most recent tag for a ref. If no ref is specified, the
# currently checked out branch is examined.
local ref="$1"
if ! git describe --abbrev=0 --first-parent ${ref} >/dev/null 2>&1; then
echo ""
else
git describe --abbrev=0 --first-parent ${ref}
fi
}
function update_gitreview {
typeset branch="$1"
title "Updating .gitreview"
git checkout $branch
# Remove a trailing newline, if present, to ensure consistent
# formatting when we add the defaultbranch line next.
typeset grcontents="$(echo -n "$(cat .gitreview | grep -v defaultbranch)")
defaultbranch=$branch"
echo "$grcontents" > .gitreview
git add .gitreview
git commit -m "Update .gitreview for $branch"
git show
local shortbranch=$(basename $branch)
git review -t "create-${shortbranch}"
}
function clone_repo {
typeset repo="$1"
typeset branch="$2"
if [ -z "$branch" ]; then
branch="master"
fi
title "Cloning $repo"
output=$({ zuul-cloner --branch "$branch" git://git.openstack.org $repo \
&& (cd $repo && git review -s); } 2>&1)
_retval=$?
if [ $_retval -ne 0 ] ; then
echo "$output"
fi
return $_retval
}

View File

@ -0,0 +1,78 @@
#!/usr/bin/env python
#
# Add a comment on a number of Launchpad bugs
#
# Copyright 2015 Thierry Carrez <thierry@openstack.org>
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import argparse
import os
import launchpadlib.launchpad
import lazr.restfulclient.errors
def main():
# Parameters
parser = argparse.ArgumentParser(description="Add comment on bugs")
parser.add_argument('--subject', help='The comment subject',
default='Comment added by add_comment')
parser.add_argument('--content', help='The comment content',
default='Comment added by add_comment')
lp_grp = parser.add_argument_group('launchpad')
lp_grp.add_argument(
"--test",
action='store_const',
dest='lp_service_root',
const='staging',
default='production',
help='Use LP staging server to test',
)
lp_grp.add_argument(
'--credentials-file', '-c',
dest='lp_creds_file',
default=os.environ.get('LP_CREDS_FILE'),
help=('plain-text credentials file, '
'defaults to value of $LP_CREDS_FILE'),
)
parser.add_argument('bugs', type=int, nargs='+',
help='Bugs to add comment to')
args = parser.parse_args()
# Connect to Launchpad
print("Connecting to Launchpad...")
launchpad = launchpadlib.launchpad.Launchpad.login_with(
application_name='openstack-releasing',
service_root=args.lp_service_root,
credentials_file=args.lp_creds_file,
)
# Add comment
for bugid in args.bugs:
print("Adding comment to #%d..." % bugid, end='')
try:
bug = launchpad.bugs[bugid]
bug.newMessage(subject=args.subject, content=args.content)
print (" done.")
except lazr.restfulclient.errors.ServerError as e:
print(" TIMEOUT during save !")
except Exception as e:
print(" ERROR during save ! (%s)" % e)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,145 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Lists the most recent versions in the deliverable files.
"""
from __future__ import print_function
import argparse
import os.path
import re
import subprocess
import yaml
PRE_RELEASE_RE = re.compile('''
\.(\d+(?:[ab]|rc)+\d*)$
''', flags=re.VERBOSE | re.UNICODE)
def find_modified_deliverable_files(reporoot):
"Return a list of files modified by the most recent commit."
results = subprocess.check_output(
['git', 'diff', '--name-only', '--pretty=format:', 'HEAD^'],
cwd=reporoot,
)
filenames = [
l.strip()
for l in results.splitlines()
if l.startswith('deliverables/')
]
return filenames
def get_modified_deliverable_file_content(reporoot, filenames):
"""Return a sequence of tuples containing the new versions.
Return tuples containing (deliverable name, series name, version
number, repository name, hash SHA, include pypi link, first full
version)
"""
# Determine which deliverable files to process by taking our
# command line arguments or by scanning the git repository
# for the most recent change.
deliverable_files = filenames
if not deliverable_files:
deliverable_files = find_modified_deliverable_files(
reporoot
)
for basename in deliverable_files:
filename = os.path.join(reporoot, basename)
if not os.path.exists(filename):
# The file must have been deleted, skip it.
continue
with open(filename, 'r') as f:
deliverable_data = yaml.load(f.read())
# Determine where to send email announcements of
# releases. Default to the development list, to cut down on
# excessive emails to the announcement list.
send_announcements_to = deliverable_data.get(
'send-announcements-to',
'openstack-dev@lists.openstack.org',
)
# Determine whether announcements should include a PyPI
# link. Default to no, for service projects, because it is
# less irksome to fail to include a link to a thing that
# exists than to link to something that does not.
include_pypi_link = deliverable_data.get(
'include-pypi-link',
False,
)
include_pypi_link = 'yes' if include_pypi_link else 'no'
# The series name is part of the filename, rather than the file
# body. That causes release.sh to be called with series="_independent"
# for release:independent projects, and release.sh to use master branch
# to evaluate fixed bugs.
series_name = os.path.basename(
os.path.dirname(os.path.abspath(filename))
).lstrip('_')
deliverable_name = os.path.splitext(os.path.basename(filename))[0]
all_versions = {
rel['version']: rel for rel in deliverable_data['releases']
}
version = deliverable_data['releases'][-1]['version']
this_version = all_versions[version]
final_versions = [
r['version']
for r in deliverable_data['releases']
if not PRE_RELEASE_RE.search(r['version'])
]
first_full_release = 'yes' if (
final_versions and
this_version['version'] == final_versions[0]
) else 'no'
for project in this_version['projects']:
yield (deliverable_name, series_name, version,
project['repo'], project['hash'],
send_announcements_to,
include_pypi_link,
first_full_release)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'deliverable_file',
nargs='*',
help='paths to YAML files specifying releases',
)
parser.add_argument(
'--releases-repo', '-r',
default='.',
help='path to the releases repository for automatic scanning',
)
args = parser.parse_args()
results = get_modified_deliverable_file_content(
args.releases_repo,
args.deliverable_file,
)
for r in results:
print(' '.join(r))
return 0
if __name__ == '__main__':
main()

View File

@ -0,0 +1,129 @@
#!/bin/bash
#
# Script to release a project in one shot, including the git tag and
# launchpad updates.
#
# Copyright 2015 Thierry Carrez <thierry@openstack.org>
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -e
TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $TOOLSDIR/functions
function usage {
echo "Usage: release.sh [-a] repository series version SHA announce include_pypi first-full-release extra-metadata"
echo
echo "Example: release.sh openstack/oslo.rootwrap mitaka 3.0.3 gerrit/master openstack-dev@lists.openstack.org yes no 'meta:release:Workflow+1: Doug Hellmann <doug@doughellmann.com>'"
}
if [ $# -lt 5 ]; then
usage
exit 2
fi
REPO=$1
SERIES=$2
VERSION=$3
SHA=$4
ANNOUNCE=$5
INCLUDE_PYPI=${6:-no}
FIRST_FULL=${7:-no}
EXTRA_METADATA="$8"
SHORTNAME=`basename $REPO`
pre_release_pat='\.[[:digit:]]+[ab][[:digit:]]+'
rc_release_pat='\.[[:digit:]]+rc[[:digit:]]+'
if [[ $VERSION =~ $pre_release_pat ]]; then
RELEASETYPE="development milestone"
elif [[ $VERSION =~ $rc_release_pat ]]; then
RELEASETYPE="release candidate"
else
RELEASETYPE="release"
fi
setup_temp_space release-tag-$SHORTNAME
clone_repo $REPO
REPODIR="$(cd $REPO && pwd)"
cd $REPODIR
TARGETSHA=`git log -1 $SHA --format='%H'`
# Determine the most recent tag before we add the new one.
PREVIOUS=$(get_last_tag $TARGETSHA)
title "Tagging $TARGETSHA as $VERSION"
if git show-ref "$VERSION"; then
echo "$REPO already has a version $VERSION tag"
# Reset the notion of "previous" to the version associated with
# the parent of the commit being tagged, since the tag we're
# applying already exists.
PREVIOUS=$(get_last_tag ${TARGETSHA}^1)
else
# WARNING(dhellmann): announce.sh expects to be able to parse this
# commit message, so if you change the format you may have to
# update announce.sh as well.
TAGMSG="$SHORTNAME $VERSION $RELEASETYPE
meta:version: $VERSION
meta:series: $SERIES
meta:release-type: $RELEASETYPE
meta:announce: $ANNOUNCE
meta:pypi: $INCLUDE_PYPI
meta:first: $FIRST_FULL
$EXTRA_METADATA
"
echo "Tag message is '$TAGMSG'"
git tag -m "$TAGMSG" -s "$VERSION" $TARGETSHA
git push gerrit $VERSION
fi
# We don't want to die just because we can't update some bug reports,
# so ignore failures.
set +e
title "Adding comments to fixed bugs"
BUGS=$(git log $PREVIOUS..$VERSION | egrep -i "Closes(.| )Bug:" | egrep -o "[0-9]+")
if [[ -z "$BUGS" ]]; then
echo "No bugs found $PREVIOUS .. $VERSION"
else
$TOOLSDIR/launchpad_add_comment.py \
--subject="Fix included in $REPO $VERSION" \
--content="This issue was fixed in the $REPO $VERSION $RELEASETYPE." \
$BUGS
fi
# Try to propose a constraints update for libraries.
if [[ $INCLUDE_PYPI == "yes" ]]; then
title "Proposing constraints update"
dist_name=$(python setup.py --name)
if [[ -z "$dist_name" ]]; then
echo "Could not determine the name of the constraint to update"
else
cd $MYTMPDIR
clone_repo openstack/requirements stable/$SERIES
cd openstack/requirements
git checkout -b "$dist_name-$VERSION"
sed -e "s/^${dist_name}=.*/$dist_name===$VERSION/" --in-place upper-constraints.txt
git commit -a -m "update constraint for $dist_name to new release $VERSION
$TAGMSG
"
git show
git review -t 'new-release'
fi
fi
exit 0

View File

@ -0,0 +1,71 @@
#!/bin/bash
#
# Script to release projects based on changes to the deliverables
# files in the openstack/releases repository.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -e
TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $TOOLSDIR/functions
function usage {
echo "Usage: release_from_yaml.sh releases_repository [deliverable_files]"
echo
echo "Example: release_from_yaml.sh ~/repos/openstack/releases"
echo "Example: release_from_yaml.sh ~/repos/openstack/releases"
echo "Example: release_from_yaml.sh ~/repos/openstack/releases deliverables/mitaka/oslo.config.yaml"
}
if [ $# -lt 1 ]; then
echo "ERROR: No releases_repository specified"
echo
usage
exit 1
fi
RELEASES_REPO="$1"
shift
DELIVERABLES="$@"
# Configure git to pull the notes where gerrit stores review history
# like who approved a patch.
cd $RELEASES_REPO
if ! git config --get remote.origin.fetch | grep -q refs/notes/review; then
git config --add remote.origin.fetch refs/notes/review:refs/notes/review
git config --add core.notesRef refs/notes/review
git remote update origin
fi
# Look for metadata about the release instructions to include in the
# tag message.
if git show --no-patch --pretty=format:%P | grep -q ' '; then
# multiple parents, look at the submitted patch instead of the
# merge commit
parent=$(git show --no-patch --pretty=format:%P | cut -f2 -d' ')
else
# single parent, look at the current patch
parent=''
fi
RELEASE_META=$(git show --format=full --show-notes=review $parent | egrep -i '(Author|Commit:|Code-Review|Workflow|Change-Id)' | sed -e 's/^ //g' -e 's/^/meta:release:/g')
$TOOLSDIR/list_deliverable_changes.py -r $RELEASES_REPO $DELIVERABLES \
| while read deliverable series version repo hash announce_to pypi first_full; do
title "$repo $series $version $hash $announce_to"
$TOOLSDIR/release.sh $repo $series $version $hash $announce_to $pypi $first_full "$RELEASE_META"
done
exit 0

View File

@ -0,0 +1,6 @@
pbr>=1.4
launchpadlib
requests>=1.1
PyYAML>=3.1.0
zuul
simplejson>=2.2.0