3f5c4ffb45
We had two separate places where we were applying the temporary tag for the new version. Consolidate that into one place, early in the processing, so the rest of the logic can assume that the tag exists. Change-Id: Id712292036abf98124bb41c6650dadbb698b942e Signed-off-by: Doug Hellmann <doug@doughellmann.com>
558 lines
20 KiB
Python
558 lines
20 KiB
Python
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""Show the changes that will be included in the release.
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import atexit
|
|
import glob
|
|
import json
|
|
import logging
|
|
import os
|
|
import os.path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
import requests
|
|
|
|
from openstack_releases import defaults
|
|
from openstack_releases import gitutils
|
|
from openstack_releases import governance
|
|
from openstack_releases import release_notes
|
|
from openstack_releases import yamlutils
|
|
|
|
|
|
def header(title):
|
|
print('\n%s' % title)
|
|
print('-' * len(title))
|
|
|
|
|
|
def git_show(workdir, repo, title, ref):
|
|
header('%s %s' % (title, ref))
|
|
cmd = ['git', 'log', '-n', '1', '--decorate', '--format=medium', ref]
|
|
print('\n' + ' '.join(cmd) + '\n')
|
|
subprocess.check_call(cmd, cwd=os.path.join(workdir, repo))
|
|
print()
|
|
|
|
|
|
def git_log(workdir, repo, title, git_range, extra_args=[]):
|
|
header('%s %s' % (title, git_range))
|
|
cmd = ['git', 'log', '--no-color']
|
|
cmd.extend(extra_args)
|
|
if isinstance(git_range, str):
|
|
cmd.append(git_range)
|
|
else:
|
|
cmd.extend(git_range)
|
|
print('\n' + ' '.join(cmd) + '\n')
|
|
subprocess.check_call(cmd, cwd=os.path.join(workdir, repo))
|
|
print()
|
|
|
|
|
|
def git_list_existing_branches(workdir, repo):
|
|
header('All Branches with Version Numbers')
|
|
for branch in gitutils.get_branches(workdir, repo):
|
|
try:
|
|
description = subprocess.check_output(
|
|
['git', 'describe', branch],
|
|
cwd=os.path.join(workdir, repo),
|
|
).decode('utf-8').strip()
|
|
except subprocess.CalledProcessError as exc:
|
|
description = exc.output
|
|
print('{:<30} {}'.format(branch, description))
|
|
|
|
|
|
def git_branch_contains(workdir, repo, title, commit):
|
|
header('%s %s' % (title, commit))
|
|
cmd = ['git', 'branch', '-r', '--contains', commit]
|
|
print('\n' + ' '.join(cmd) + '\n')
|
|
out = subprocess.check_output(
|
|
cmd,
|
|
cwd=os.path.join(workdir, repo),
|
|
).decode('utf-8')
|
|
print(out)
|
|
return sorted(
|
|
o.strip()
|
|
for o in out.splitlines()
|
|
if '->' not in o
|
|
)
|
|
|
|
|
|
def git_diff(workdir, repo, git_range, file_pattern):
|
|
repo_dir = os.path.join(workdir, repo)
|
|
files = list(glob.glob(os.path.join(repo_dir,
|
|
file_pattern)))
|
|
if files:
|
|
header('Requirements Changes %s' % git_range)
|
|
for f in files:
|
|
cmd = [
|
|
'git', 'diff', '-U0', '--no-color',
|
|
'--ignore-space-change', '--ignore-blank-lines',
|
|
git_range,
|
|
f[len(repo_dir) + 1:],
|
|
]
|
|
print(' '.join(cmd) + '\n')
|
|
subprocess.check_call(cmd, cwd=repo_dir)
|
|
print()
|
|
|
|
|
|
def gerrit_query(*query):
|
|
url = 'https://review.openstack.org/changes/?q=' + '+'.join(query)
|
|
response = requests.get(url)
|
|
if (response.status_code // 100) != 2:
|
|
raise RuntimeError(
|
|
'Bad HTTP response from gerrit %s: %s' %
|
|
(url, response.status_code)
|
|
)
|
|
elif response.content[:4] == b")]}'":
|
|
content = response.content[5:].decode('utf-8')
|
|
return json.loads(content)
|
|
else:
|
|
print('could not parse response from %s' % url)
|
|
print(repr(content))
|
|
raise RuntimeError('failed to parse gerrit response')
|
|
|
|
|
|
def list_gerrit_patches(title, template, query):
|
|
header('{}: "{}"'.format(title, query))
|
|
try:
|
|
reviews = gerrit_query(query)
|
|
except Exception as err:
|
|
print(err)
|
|
else:
|
|
for r in reviews:
|
|
if 'topic' not in r:
|
|
r['topic'] = ''
|
|
try:
|
|
print(template.format(**r))
|
|
except Exception as err:
|
|
print('Could not format review data: {}'.format(err))
|
|
print(r)
|
|
print('{} results\n'.format(len(reviews)))
|
|
|
|
|
|
def show_watched_queries(branch, repo):
|
|
with open('watched_queries.yml', 'r', encoding='utf-8') as f:
|
|
watched_queries = yamlutils.loads(f.read())
|
|
template = watched_queries['template']
|
|
for q in watched_queries['queries']:
|
|
list_gerrit_patches(
|
|
q['title'],
|
|
q.get('template', template),
|
|
q['query'].format(
|
|
branch=branch,
|
|
project=repo,
|
|
),
|
|
)
|
|
|
|
|
|
def main():
|
|
if not sys.stdout.encoding:
|
|
# Wrap sys.stdout with a writer that knows how to handle
|
|
# encoding Unicode data.
|
|
import codecs
|
|
wrapped_stdout = codecs.getwriter('UTF-8')(sys.stdout)
|
|
sys.stdout = wrapped_stdout
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
'--no-cleanup',
|
|
dest='cleanup',
|
|
default=True,
|
|
action='store_false',
|
|
help='do not remove temporary files',
|
|
)
|
|
parser.add_argument(
|
|
'input',
|
|
nargs='*',
|
|
help=('YAML files to validate, defaults to '
|
|
'files changed in the latest commit'),
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
# Set up logging, including making some loggers quiet.
|
|
logging.basicConfig(
|
|
format='%(levelname)7s:%(name)s: %(message)s',
|
|
stream=sys.stdout,
|
|
level=logging.DEBUG,
|
|
)
|
|
logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING)
|
|
|
|
filenames = args.input or gitutils.find_modified_deliverable_files()
|
|
if not filenames:
|
|
print('no modified deliverable files, skipping report')
|
|
return 0
|
|
|
|
workdir = tempfile.mkdtemp(prefix='releases-')
|
|
print('creating temporary files in %s' % workdir)
|
|
|
|
def cleanup_workdir():
|
|
if args.cleanup:
|
|
try:
|
|
shutil.rmtree(workdir)
|
|
except:
|
|
pass
|
|
else:
|
|
print('not cleaning up %s' % workdir)
|
|
atexit.register(cleanup_workdir)
|
|
|
|
team_data = governance.get_team_data()
|
|
|
|
# Remove any inherited PAGER environment variable to avoid
|
|
# blocking the output waiting for input.
|
|
os.environ['PAGER'] = ''
|
|
|
|
for filename in filenames:
|
|
if not os.path.exists(filename):
|
|
print('\n%s was removed, skipping' % filename)
|
|
continue
|
|
print('\n' + ('=' * 80))
|
|
print('\nChecking %s\n' % filename)
|
|
with open(filename, 'r', encoding='utf-8') as f:
|
|
deliverable_info = yamlutils.loads(f.read())
|
|
|
|
series = os.path.basename(
|
|
os.path.dirname(
|
|
os.path.abspath(filename)
|
|
)
|
|
)
|
|
if series == '_independent':
|
|
default_model = 'independent'
|
|
else:
|
|
default_model = 'no release model specified'
|
|
|
|
# By default assume the project does not use milestones.
|
|
header('Release model')
|
|
print(deliverable_info.get('release-model', default_model))
|
|
|
|
header('Team details')
|
|
if 'team' in deliverable_info:
|
|
team_name = deliverable_info['team']
|
|
team_dict = team_data.get(team_name)
|
|
if team_dict:
|
|
team = governance.Team(team_name, team_dict)
|
|
print('found team %s' % team_name)
|
|
print(' PTL : %(name)s (%(irc)s)' % team.ptl)
|
|
print(' Liaison: %s (%s)\n' % team.liaison)
|
|
deliverable_name = os.path.basename(filename)[:-5] # remove .yaml
|
|
deliverable = team.deliverables.get(deliverable_name)
|
|
if deliverable:
|
|
print('found deliverable %s' % deliverable_name)
|
|
for rn, repo in sorted(deliverable.repositories.items()):
|
|
print('\nrepo %s\ntags:' % repo.name)
|
|
for t in repo.tags:
|
|
print(' %s' % t)
|
|
print('')
|
|
else:
|
|
print(('no deliverable %r found for team %r, '
|
|
'cannot report on governance status') %
|
|
(deliverable_name, team_name))
|
|
else:
|
|
print('no team %r found, cannot report on governance status' %
|
|
team_name)
|
|
else:
|
|
print('no team name given, cannot report on governance status')
|
|
|
|
# If there are no releases listed, this is probably a new
|
|
# deliverable file for initializing a new series. We don't
|
|
# need to list its changes.
|
|
if not deliverable_info.get('releases'):
|
|
header('No releases')
|
|
print('no releases were found, assuming an initialization file')
|
|
continue
|
|
|
|
# assume the releases are in order and take the last one
|
|
new_release = deliverable_info['releases'][-1]
|
|
|
|
# build a map between version numbers and the release details
|
|
by_version = {
|
|
str(r['version']): r
|
|
for r in deliverable_info['releases']
|
|
}
|
|
|
|
for project in new_release['projects']:
|
|
|
|
tag_exists = gitutils.tag_exists(
|
|
project['repo'],
|
|
new_release['version'],
|
|
)
|
|
if tag_exists:
|
|
print('%s %s exists on git server already' %
|
|
(project['repo'], new_release['version']))
|
|
|
|
# Decide which branch we're going to try to clone. We need
|
|
# the repo checked out before we can tell if the stable
|
|
# branch really exists, but zuul-cloner will fall back to
|
|
# master if it doesn't.
|
|
if series in (defaults.RELEASE, '_independent'):
|
|
clone_branch = 'master'
|
|
else:
|
|
clone_branch = 'stable/' + series
|
|
|
|
# Check out the code.
|
|
print('\nChecking out repository {} to {}'.format(
|
|
project['repo'], clone_branch))
|
|
subprocess.check_call(
|
|
['zuul-cloner',
|
|
'--branch', clone_branch,
|
|
'--workspace', workdir,
|
|
'git://git.openstack.org',
|
|
project['repo'],
|
|
]
|
|
)
|
|
# Set some git configuration values to allow us to perform
|
|
# local operations like tagging.
|
|
gitutils.ensure_basic_git_config(
|
|
workdir, project['repo'],
|
|
{'user.email': 'openstack-infra@lists.openstack.org',
|
|
'user.name': 'OpenStack Proposal Bot'},
|
|
)
|
|
|
|
# Determine which branch we should actually be looking
|
|
# at. Assume any series for which there is no stable
|
|
# branch will be on 'master'.
|
|
if gitutils.stable_branch_exists(workdir, project['repo'], series):
|
|
branch = 'stable/' + series
|
|
else:
|
|
branch = 'master'
|
|
|
|
if branch != clone_branch:
|
|
# Check out the repo again to the right branch if we
|
|
# didn't get it the first time.
|
|
print('\nUpdating repository {} to {}'.format(
|
|
project['repo'], branch))
|
|
subprocess.check_call(
|
|
['zuul-cloner',
|
|
'--branch', branch,
|
|
'--workspace', workdir,
|
|
'git://git.openstack.org',
|
|
project['repo'],
|
|
]
|
|
)
|
|
|
|
# look at the previous tag for the parent of the commit
|
|
# getting the new release
|
|
previous_tag = gitutils.get_latest_tag(
|
|
workdir,
|
|
project['repo'],
|
|
'{}^'.format(project['hash'])
|
|
)
|
|
previous_release = by_version.get(previous_tag)
|
|
|
|
start_range = previous_tag
|
|
if previous_release:
|
|
previous_project = {
|
|
x['repo']: x
|
|
for x in previous_release['projects']
|
|
}.get(project['repo'])
|
|
if previous_project is not None:
|
|
start_range = previous_tag
|
|
|
|
if start_range:
|
|
git_range = '%s..%s' % (start_range, project['hash'])
|
|
else:
|
|
git_range = project['hash']
|
|
|
|
# Show details about the commit being tagged.
|
|
header('Details for commit receiving new tag %s' %
|
|
new_release['version'])
|
|
print('\ngit describe %s\n' % project['hash'])
|
|
try:
|
|
subprocess.check_call(
|
|
['git', 'describe', project['hash']],
|
|
cwd=os.path.join(workdir, project['repo']),
|
|
)
|
|
except subprocess.CalledProcessError as e:
|
|
print('WARNING: Could not run git describe: %s' % e)
|
|
|
|
git_show(
|
|
workdir=workdir,
|
|
repo=project['repo'],
|
|
title='Check existing tags',
|
|
ref=project['hash'],
|
|
)
|
|
|
|
git_list_existing_branches(
|
|
workdir=workdir,
|
|
repo=project['repo'],
|
|
)
|
|
|
|
branches = git_branch_contains(
|
|
workdir=workdir,
|
|
repo=project['repo'],
|
|
title='Branches containing commit',
|
|
commit=project['hash'],
|
|
)
|
|
|
|
header('Relationship to HEAD')
|
|
if series == '_independent':
|
|
if branches:
|
|
tag_branch = branches[0]
|
|
else:
|
|
tag_branch = branch
|
|
head_sha = gitutils.sha_for_tag(
|
|
workdir,
|
|
project['repo'],
|
|
tag_branch,
|
|
)
|
|
print('HEAD of {} is {}'.format(tag_branch, head_sha))
|
|
else:
|
|
if (branch in branches) or (not branches):
|
|
tag_branch = branch
|
|
else:
|
|
tag_branch = branches[0]
|
|
head_sha = gitutils.sha_for_tag(
|
|
workdir,
|
|
project['repo'],
|
|
tag_branch,
|
|
)
|
|
print('HEAD of {} is {}'.format(tag_branch, head_sha))
|
|
requested_sha = gitutils.sha_for_tag(
|
|
workdir,
|
|
project['repo'],
|
|
project['hash'],
|
|
)
|
|
# If the sha for HEAD and the requested release don't
|
|
# match, show any unreleased changes on the branch. We ask
|
|
# git to give us the real SHA for the requested release in
|
|
# case the deliverables file has the short version of the
|
|
# hash.
|
|
if head_sha == requested_sha:
|
|
print('\nRequest releases from HEAD on %s' % tag_branch)
|
|
else:
|
|
git_log(workdir, project['repo'], 'Release will NOT include',
|
|
'%s..%s' % (requested_sha, head_sha),
|
|
extra_args=['--format=%h %ci %s'])
|
|
|
|
show_watched_queries(branch, project['repo'])
|
|
|
|
# Show any requirements changes in the upcoming release.
|
|
# Include setup.cfg, in case the project uses "extras".
|
|
if start_range:
|
|
git_diff(workdir, project['repo'], git_range, '*requirements*.txt')
|
|
git_diff(workdir, project['repo'], git_range, 'setup.cfg')
|
|
|
|
# Before we try to determine if the previous release
|
|
# is an ancestor or produce the release notes we need
|
|
# the tag to exist in the local repository.
|
|
if not tag_exists:
|
|
header('Applying Temporary Tag')
|
|
print('\ngit tag {version} {hash}'.format(
|
|
version=new_release['version'],
|
|
hash=project['hash'],
|
|
))
|
|
subprocess.check_call(
|
|
['git', 'tag', new_release['version'],
|
|
project['hash']],
|
|
cwd=os.path.join(workdir, project['repo']),
|
|
)
|
|
|
|
# Show any changes in the previous release but not in this
|
|
# release, in case someone picks an "early" SHA or a
|
|
# regular commit instead of the appropriate merge commit.
|
|
previous_tag_exists = False
|
|
if previous_release:
|
|
previous_tag_exists = gitutils.tag_exists(
|
|
project['repo'],
|
|
previous_release['version'],
|
|
)
|
|
if previous_tag_exists:
|
|
git_log(
|
|
workdir, project['repo'],
|
|
'Patches in previous release but not in this one',
|
|
[previous_release['version'],
|
|
'--not',
|
|
project['hash']],
|
|
extra_args=['--topo-order', '--oneline', '--no-merges'],
|
|
)
|
|
|
|
# The tag will have been added as a local tag above if
|
|
# it does not already exist.
|
|
header('New release %s includes previous release %s' %
|
|
(new_release['version'], previous_release['version']))
|
|
print('\ngit tag --contains %s\n' %
|
|
previous_release['version'])
|
|
containing_tags = subprocess.check_output(
|
|
['git', 'tag',
|
|
'--contains',
|
|
previous_release['version']],
|
|
cwd=os.path.join(workdir, project['repo']),
|
|
).decode('utf-8').split()
|
|
print('Containing tags:', containing_tags)
|
|
if new_release['version'] not in containing_tags:
|
|
print('WARNING: Missing %s' % new_release['version'])
|
|
else:
|
|
print('Found new version %s' % new_release['version'])
|
|
|
|
is_ancestor = gitutils.check_ancestry(
|
|
workdir,
|
|
project['repo'],
|
|
previous_release['version'],
|
|
project['hash'],
|
|
)
|
|
if is_ancestor:
|
|
print('SHA found in descendants')
|
|
else:
|
|
print('SHA NOT FOUND in descendants')
|
|
|
|
# Show the changes since the last release, first as a
|
|
# graph view so we can check for bad merges, and then with
|
|
# more detail.
|
|
git_log(workdir, project['repo'],
|
|
'Release %s will include' % new_release['version'],
|
|
git_range,
|
|
extra_args=['--graph', '--oneline', '--decorate',
|
|
'--topo-order'])
|
|
git_log(workdir, project['repo'],
|
|
'Details Contents',
|
|
git_range,
|
|
extra_args=['--no-merges', '--topo-order'])
|
|
|
|
# The tag will have been added as a local tag above if it does
|
|
# not already exist.
|
|
header('Release Notes')
|
|
try:
|
|
first_release = len(deliverable_info.get('releases', [])) == 1
|
|
notes = release_notes.generate_release_notes(
|
|
repo=project['repo'],
|
|
repo_path=os.path.join(workdir, project['repo']),
|
|
start_revision=start_range,
|
|
end_revision=new_release['version'],
|
|
show_dates=True,
|
|
skip_requirement_merges=True,
|
|
is_stable=branch.startswith('stable/'),
|
|
series=series,
|
|
email='test-job@openstack.org',
|
|
email_from='test-job@openstack.org',
|
|
email_reply_to='noreply@openstack.org',
|
|
email_tags='',
|
|
include_pypi_link=False,
|
|
changes_only=False,
|
|
first_release=first_release,
|
|
repo_name=project['repo'],
|
|
description='',
|
|
publishing_dir_name=project['repo'],
|
|
)
|
|
except Exception as e:
|
|
logging.exception('Failed to produce release notes')
|
|
else:
|
|
print('\n')
|
|
print(notes)
|
|
|
|
return 0
|