releases/openstack_releases/cmds/list_changes.py

617 lines
22 KiB
Python

# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Show the changes that will be included in the release.
"""
import argparse
import atexit
import glob
import json
import logging
import os
import os.path
import re
import shutil
import subprocess
import sys
import tempfile
from openstack_governance import governance
import pyfiglet
import requests
from openstack_releases import defaults
from openstack_releases import deliverable
from openstack_releases import gitutils
from openstack_releases import hound
from openstack_releases import liaisons
from openstack_releases import release_notes
from openstack_releases import yamlutils
def header(title):
print('\n%s' % title)
print('-' * len(title))
def banner(text):
pyfiglet.print_figlet(text, font='banner', width=120)
def git_show(workdir, repo, title, ref):
header('%s %s' % (title, ref))
cmd = ['git', 'log', '-n', '1', '--decorate', '--format=medium', ref]
print('\n' + ' '.join(cmd) + '\n')
subprocess.check_call(cmd, cwd=os.path.join(workdir, repo))
print()
def git_log(workdir, repo, title, git_range, extra_args=[]):
header('%s %s' % (title, git_range))
cmd = ['git', 'log', '--no-color']
cmd.extend(extra_args)
if isinstance(git_range, str):
cmd.append(git_range)
else:
cmd.extend(git_range)
print('\n' + ' '.join(cmd) + '\n')
subprocess.check_call(cmd, cwd=os.path.join(workdir, repo))
print()
def git_list_existing_branches(workdir, repo):
header('All Branches with Version Numbers')
for branch in gitutils.get_branches(workdir, repo):
try:
description = subprocess.check_output(
['git', 'describe', branch],
cwd=os.path.join(workdir, repo),
).decode('utf-8').strip()
# strip to the real tag value
match = re.match('^(.*)-[0-9]+-g[a-f0-9]+', description,
re.IGNORECASE)
if match:
tag = match.groups()[0]
else:
tag = ''
except subprocess.CalledProcessError as exc:
description = exc.output.decode('utf-8').strip()
tag = ''
if not tag:
print('{:<30} {:<20}'.format(branch, description))
else:
try:
date = subprocess.check_output(
['git', 'log', '-1', '--pretty=format:%ar', tag],
cwd=os.path.join(workdir, repo),
).decode('utf-8').strip()
except subprocess.CalledProcessError as exc:
date = exc.output.decode('utf-8')
print('{:<30} {:<20} {:<12} {}'.format(
branch, description, tag, date))
def git_branch_contains(workdir, repo, title, commit):
header('%s %s' % (title, commit))
cmd = ['git', 'branch', '-r', '--contains', commit]
print('\n' + ' '.join(cmd) + '\n')
out = subprocess.check_output(
cmd,
cwd=os.path.join(workdir, repo),
).decode('utf-8')
print(out)
return sorted(
o.strip()
for o in out.splitlines()
if '->' not in o
)
def git_diff(workdir, repo, git_range, file_pattern, title=''):
repo_dir = os.path.join(workdir, repo)
files = list(glob.glob(os.path.join(repo_dir,
file_pattern)))
if files:
if title:
header(title)
for f in files:
cmd = [
'git', 'diff', '-U0', '--no-color',
'--ignore-space-change', '--ignore-blank-lines',
git_range,
f[len(repo_dir) + 1:],
]
print(' '.join(cmd) + '\n')
subprocess.check_call(cmd, cwd=repo_dir)
print()
def gerrit_query(*query):
url = 'https://review.openstack.org/changes/?q=' + '+'.join(query)
response = requests.get(url)
if (response.status_code // 100) != 2:
raise RuntimeError(
'Bad HTTP response from gerrit %s: %s' %
(url, response.status_code)
)
elif response.content[:4] == b")]}'":
content = response.content[5:].decode('utf-8')
return json.loads(content)
else:
print('could not parse response from %s' % url)
print(repr(content))
raise RuntimeError('failed to parse gerrit response')
def list_gerrit_patches(title, template, query):
header('{}: "{}"'.format(title, query))
try:
reviews = gerrit_query(query)
except Exception as err:
print(err)
else:
for r in reviews:
if 'topic' not in r:
r['topic'] = ''
try:
print(template.format(**r))
except Exception as err:
print('Could not format review data: {}'.format(err))
print(r)
print('{} results\n'.format(len(reviews)))
def show_watched_queries(branch, repo):
with open('watched_queries.yml', 'r', encoding='utf-8') as f:
watched_queries = yamlutils.loads(f.read())
template = watched_queries['template']
for q in watched_queries['queries']:
list_gerrit_patches(
q['title'],
q.get('template', template),
q['query'].format(
branch=branch,
project=repo,
),
)
def show_dependency_listings(package_name, official_repos):
header('Users of {}'.format(package_name))
hound.show_dependency_listings(package_name, official_repos)
def main():
if not sys.stdout.encoding:
# Wrap sys.stdout with a writer that knows how to handle
# encoding Unicode data.
import codecs
wrapped_stdout = codecs.getwriter('UTF-8')(sys.stdout)
sys.stdout = wrapped_stdout
parser = argparse.ArgumentParser()
parser.add_argument(
'--no-cleanup',
dest='cleanup',
default=True,
action='store_false',
help='do not remove temporary files',
)
parser.add_argument(
'--no-shortcut', '--force', '-f',
dest='shortcut',
default=True,
action='store_false',
help='if a tag has been applied, skip the repo',
)
parser.add_argument(
'input',
nargs='*',
help=('YAML files to validate, defaults to '
'files changed in the latest commit'),
)
args = parser.parse_args()
# Set up logging, including making some loggers quiet.
logging.basicConfig(
format='%(levelname)7s: %(message)s',
stream=sys.stdout,
level=logging.DEBUG,
)
logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING)
filenames = args.input or gitutils.find_modified_deliverable_files()
if not filenames:
print('no modified deliverable files, skipping report')
return 0
workdir = tempfile.mkdtemp(prefix='releases-')
print('creating temporary files in %s' % workdir)
def cleanup_workdir():
if args.cleanup:
shutil.rmtree(workdir, True)
else:
print('not cleaning up %s' % workdir)
atexit.register(cleanup_workdir)
gov_data = governance.Governance.from_remote_repo()
official_repos = set(
r.name
for r in gov_data.get_repositories()
)
all_deliverables = deliverable.Deliverables(
'./deliverables',
False,
)
liaison_data = liaisons.get_liaisons()
# Remove any inherited PAGER environment variable to avoid
# blocking the output waiting for input.
os.environ['PAGER'] = ''
for filename in filenames:
if not os.path.exists(filename):
print('\n%s was removed, skipping' % filename)
continue
print('\n' + ('=' * 80))
print('\nChecking %s\n' % filename)
deliv = deliverable.Deliverable.read_file(filename)
stable_branch = deliv.series not in ['independent', defaults.RELEASE]
# By default assume the project does not use milestones.
header('Release model')
print(deliv.model)
header('Team details')
if deliv.team:
team_name = deliv.team
try:
team = gov_data.get_team(team_name)
except ValueError:
team = None
if team:
print('found team %s' % team_name)
print(' PTL : %(name)s (%(irc)s)' % team.ptl)
for liaison in liaison_data.get(team.name.lower(), []):
print(' Liaison: %(name)s (%(irc)s)' % liaison)
team_deliv = team.deliverables.get(deliv.name)
if team_deliv:
print('found deliverable %s' % deliv.name)
for rn, repo in sorted(team_deliv.repositories.items()):
follows_stable_policy = 'stable:follows-policy' in repo.tags
print('\nrepo %s\ntags:' % repo.name)
for t in repo.tags:
print(' %s' % t)
print('')
if stable_branch and follows_stable_policy:
banner('Needs Stable Policy Review')
print()
else:
print(('no deliverable %r found for team %r, '
'cannot report on governance status') %
(deliv.name, team_name))
if not deliv.is_independent:
# Show other deliverables owned by the team and
# included in this series.
team_deliv_in_series = all_deliverables.get_deliverables(
team.name, deliv.series)
if team_deliv_in_series:
print('Other {} deliverables in {}:'.format(
team.name, deliv.series))
for d in team_deliv_in_series:
print(' {} ({})'.format(d.name, d.latest_release or None))
print()
else:
print('no team %r found, cannot report on governance status' %
team_name)
else:
print('no team name given, cannot report on governance status')
# If there are no releases listed, this is probably a new
# deliverable file for initializing a new series. We don't
# need to list its changes.
if not deliv.is_released:
header('No releases')
print('no releases were found, assuming an initialization file')
continue
# assume the releases are in order and take the last one
new_release = deliv.releases[-1]
for project in new_release.projects:
tag_exists = gitutils.tag_exists(
project.repo.name,
new_release.version,
)
if tag_exists:
print('%s %s exists on git server already' %
(project.repo.name, new_release.version))
if args.shortcut:
print('skipping further processing')
continue
if project.repo.is_retired:
print('%s is retired' % (project.repo.name,))
if args.shortcut:
print('skipping further processing')
continue
# Start by checking out master, always. We need the repo
# checked out before we can tell if the stable branch
# really exists.
gitutils.clone_repo(
workdir,
project.repo.name,
branch='master',
)
# Set some git configuration values to allow us to perform
# local operations like tagging.
gitutils.ensure_basic_git_config(
workdir, project.repo.name,
{'user.email': 'openstack-infra@lists.openstack.org',
'user.name': 'OpenStack Proposal Bot'},
)
# Determine which branch we should actually be looking
# at. Assume any series for which there is no stable
# branch will be on 'master'.
if gitutils.stable_branch_exists(workdir,
project.repo.name,
deliv.series):
branch = 'stable/' + deliv.series
else:
branch = 'master'
if branch != 'master':
# Check out the repo again to the right branch if we
# didn't get it the first time.
gitutils.clone_repo(
workdir,
project.repo.name,
branch=branch,
)
# look at the previous tag for the parent of the commit
# getting the new release
previous_tag = gitutils.get_latest_tag(
workdir,
project.repo.name,
'{}^'.format(project.hash),
always=False,
)
try:
previous_release = deliv.get_release(previous_tag)
except ValueError:
previous_release = None
start_range = previous_tag
if previous_release:
previous_project = {
x.repo.name: x
for x in previous_release.projects
}.get(project.repo.name)
if previous_project is not None:
start_range = previous_tag
if start_range:
git_range = '%s..%s' % (start_range, project.hash)
else:
git_range = project.hash
# Show details about the commit being tagged.
header('Details for commit receiving new tag %s' %
new_release.version)
print('\ngit describe %s\n' % project.hash)
try:
subprocess.check_call(
['git', 'describe', project.hash],
cwd=os.path.join(workdir, project.repo.name),
)
except subprocess.CalledProcessError as e:
print('WARNING: Could not run git describe: %s' % e)
git_show(
workdir=workdir,
repo=project.repo.name,
title='Check existing tags',
ref=project.hash,
)
git_list_existing_branches(
workdir=workdir,
repo=project.repo.name,
)
branches = git_branch_contains(
workdir=workdir,
repo=project.repo.name,
title='Branches containing commit',
commit=project.hash,
)
header('Relationship to HEAD')
if deliv.is_independent:
if branches:
tag_branch = branches[0]
else:
tag_branch = branch
head_sha = gitutils.sha_for_tag(
workdir,
project.repo.name,
tag_branch,
)
print('HEAD of {} is {}'.format(tag_branch, head_sha))
else:
if (branch in branches) or (not branches):
tag_branch = branch
else:
tag_branch = branches[0]
head_sha = gitutils.sha_for_tag(
workdir,
project.repo.name,
tag_branch,
)
print('HEAD of {} is {}'.format(tag_branch, head_sha))
requested_sha = gitutils.sha_for_tag(
workdir,
project.repo.name,
project.hash,
)
# If the sha for HEAD and the requested release don't
# match, show any unreleased changes on the branch. We ask
# git to give us the real SHA for the requested release in
# case the deliverables file has the short version of the
# hash.
if head_sha == requested_sha:
print('\nRequest releases from HEAD on %s' % tag_branch)
else:
git_log(workdir, project.repo.name, 'Release will NOT include',
'%s..%s' % (requested_sha, head_sha),
extra_args=['--format=%h %ci %s'])
show_watched_queries(branch, project.repo.name)
# Show any requirements changes in the upcoming release.
# Include setup.cfg, in case the project uses "extras".
if start_range:
git_diff(workdir, project.repo.name, git_range, '*requirements*.txt',
'Requirements Changes %s' % git_range)
git_diff(workdir, project.repo.name, git_range, 'doc/requirements.txt',
'Doc Requirements Changes %s' % git_range)
git_diff(workdir, project.repo.name, git_range, 'setup.cfg',
'setup.cfg Changes %s' % git_range)
git_diff(workdir, project.repo.name, git_range, 'bindep.txt',
'bindep.txt Changes %s' % git_range)
# Before we try to determine if the previous release
# is an ancestor or produce the release notes we need
# the tag to exist in the local repository.
if not tag_exists:
header('Applying Temporary Tag')
print('\ngit tag {version} {hash}'.format(
version=new_release.version,
hash=project.hash,
))
subprocess.check_call(
['git', 'tag', new_release.version,
project.hash],
cwd=os.path.join(workdir, project.repo.name),
)
# Show any changes in the previous release but not in this
# release, in case someone picks an "early" SHA or a
# regular commit instead of the appropriate merge commit.
previous_tag_exists = False
if previous_release:
previous_tag_exists = gitutils.tag_exists(
project.repo.name,
previous_release.version,
)
if previous_tag_exists:
git_log(
workdir, project.repo.name,
'Patches in previous release but not in this one',
[previous_release.version,
'--not',
project.hash],
extra_args=['--topo-order', '--oneline', '--no-merges'],
)
# The tag will have been added as a local tag above if
# it does not already exist.
header('New release %s includes previous release %s' %
(new_release.version, previous_release.version))
print('\ngit tag --contains %s\n' %
previous_release.version)
containing_tags = subprocess.check_output(
['git', 'tag',
'--contains',
previous_release.version],
cwd=os.path.join(workdir, project.repo.name),
).decode('utf-8').split()
print('Containing tags:', containing_tags)
if new_release.version not in containing_tags:
print('WARNING: Missing %s' % new_release.version)
else:
print('Found new version %s' % new_release.version)
is_ancestor = gitutils.check_ancestry(
workdir,
project.repo.name,
previous_release.version,
project.hash,
)
if is_ancestor:
print('SHA found in descendants')
else:
print('SHA NOT FOUND in descendants')
# Show the changes since the last release, first as a
# graph view so we can check for bad merges, and then with
# more detail.
git_log(workdir, project.repo.name,
'Release %s will include' % new_release.version,
git_range,
extra_args=['--graph', '--oneline', '--decorate',
'--topo-order'])
git_log(workdir, project.repo.name,
'Details Contents',
git_range,
extra_args=['--no-merges', '--topo-order'])
# The tag will have been added as a local tag above if it does
# not already exist.
header('Release Notes')
try:
notes = release_notes.generate_release_notes(
repo=project.repo.name,
repo_path=os.path.join(workdir, project.repo.name),
start_revision=new_release.diff_start or start_range or '',
end_revision=new_release.version,
show_dates=True,
skip_requirement_merges=True,
is_stable=branch.startswith('stable/'),
series=deliv.series,
email='test-job@openstack.org',
email_from='test-job@openstack.org',
email_reply_to='noreply@openstack.org',
email_tags='',
include_pypi_link=False,
changes_only=False,
first_release=deliv.is_first_release,
deliverable_file=filename,
description='',
publishing_dir_name=project.repo.name,
)
except Exception as e:
logging.exception('Failed to produce release notes')
else:
print('\n')
print(notes)
if 'library' in deliv.type:
show_dependency_listings(
project.guess_sdist_name(),
official_repos,
)
return 0