Merge "Allow list_unreleased_changes to format results at json & yaml formats"
This commit is contained in:
Zuul
Gerrit Code Review
commit
ca7d97f823
|
|
@ -0,0 +1,305 @@
|
|||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Show the changes that will be included in the release.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import os.path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import requests
|
||||
|
||||
from openstack_releases import gitutils
|
||||
from openstack_releases import yamlutils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def git_log(workdir, repo, git_range, extra_args=[]):
|
||||
results = {
|
||||
'range': git_range,
|
||||
'logs': []
|
||||
}
|
||||
cmd = ['git', 'log', '--no-color']
|
||||
cmd.extend(extra_args)
|
||||
if isinstance(git_range, str):
|
||||
cmd.append(git_range)
|
||||
elif isinstance(git_range, tuple):
|
||||
cmd.append('{start}..{end}'.format(
|
||||
start=git_range[0], end=git_range[1]))
|
||||
else:
|
||||
cmd.extend(git_range)
|
||||
LOG.debug('\n' + ' '.join(cmd) + '\n')
|
||||
output = subprocess.check_output(cmd, cwd=os.path.join(workdir, repo))
|
||||
results['logs'] = [el for el in output.decode('utf-8').split("\n") if el]
|
||||
LOG.debug(results)
|
||||
return results
|
||||
|
||||
|
||||
def filter_results(content,
|
||||
ignore_no_results=False,
|
||||
ignore_errors=False,
|
||||
ignore_not_yet_released=False,
|
||||
ignore_all=False):
|
||||
filtered = []
|
||||
for repo in content:
|
||||
removed = False
|
||||
if ignore_all and \
|
||||
(repo['error'] or
|
||||
repo['not_yet_released'] or
|
||||
not repo['commits'] or
|
||||
not repo['commits'].get('logs', None)):
|
||||
removed = True
|
||||
LOG.debug("ignoring all")
|
||||
elif ignore_errors and repo['error']:
|
||||
removed = True
|
||||
LOG.debug("ignoring errors")
|
||||
elif ignore_not_yet_released and repo['not_yet_released']:
|
||||
removed = True
|
||||
LOG.debug("ignoring not yet released")
|
||||
elif ignore_no_results and (not repo.get('commits', None) or
|
||||
not repo.get('commits', None).get(
|
||||
'logs', None)):
|
||||
LOG.debug("ignoring no results")
|
||||
removed = True
|
||||
if not removed:
|
||||
filtered.append(repo)
|
||||
else:
|
||||
LOG.debug('repo {} will be ignored'.format(repo['repo']))
|
||||
LOG.debug(repo)
|
||||
return filtered
|
||||
|
||||
|
||||
def generate_output(content, output_format='std',
|
||||
ignore_no_results=False,
|
||||
ignore_errors=False,
|
||||
ignore_not_yet_released=False,
|
||||
ignore_all=False):
|
||||
filtered = filter_results(content, ignore_no_results, ignore_errors,
|
||||
ignore_not_yet_released, ignore_all)
|
||||
if output_format == 'json':
|
||||
return json.dumps(filtered, indent=4)
|
||||
if output_format == 'yaml':
|
||||
return yamlutils.dumps(filtered)
|
||||
if output_format == 'std':
|
||||
out = []
|
||||
for repo in filtered:
|
||||
out.append('[ Unreleased changes in {rep} ({br}) ]'.format(
|
||||
rep=repo['repo'], br=repo['branch']))
|
||||
|
||||
if repo['commits'].get('logs', None):
|
||||
range_msg = 'Changes between {start} and {end}'.format(
|
||||
start=repo['commits']['range'][0],
|
||||
end=repo['commits']['range'][1])
|
||||
out.append(range_msg)
|
||||
out.append("\n".join(repo['commits']['logs']))
|
||||
return '\n'.join(out)
|
||||
|
||||
|
||||
def main():
|
||||
if not sys.stdout.encoding:
|
||||
# Wrap sys.stdout with a writer that knows how to handle
|
||||
# encoding Unicode data.
|
||||
import codecs
|
||||
wrapped_stdout = codecs.getwriter('UTF-8')(sys.stdout)
|
||||
sys.stdout = wrapped_stdout
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'--no-cleanup',
|
||||
dest='cleanup',
|
||||
default=True,
|
||||
action='store_false',
|
||||
help='do not remove temporary files',
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--verbosity",
|
||||
action="count",
|
||||
help="increase output verbosity",
|
||||
default=0)
|
||||
parser.add_argument(
|
||||
"--ignore-no-results",
|
||||
action='store_true',
|
||||
default=False,
|
||||
help="Ignore projects without difference between the HEAD and"
|
||||
"the retrieved previous tag."
|
||||
"They will be ignored in the command output.")
|
||||
parser.add_argument(
|
||||
"--ignore-errors",
|
||||
action='store_true',
|
||||
default=False,
|
||||
help="Ignore projects in error (repos not found).")
|
||||
parser.add_argument(
|
||||
"--ignore-not-yet-released",
|
||||
action='store_true',
|
||||
default=False,
|
||||
help="Ignore projects not yet released (previous tag not found).")
|
||||
parser.add_argument(
|
||||
"-I",
|
||||
"--ignore-all",
|
||||
action='store_true',
|
||||
default=False,
|
||||
help="Ignore projects without difference between the HEAD and"
|
||||
"previous tag, projects not yet released, projects in error."
|
||||
"Similar to call command with "
|
||||
"`--ignore-no-results --ignore-errors --ignore-not-yet-released`"
|
||||
"They will be ignored in the command output.")
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--format",
|
||||
choices=['std', 'json', 'yaml'],
|
||||
default='standard',
|
||||
help="Output format")
|
||||
parser.add_argument(
|
||||
'branch',
|
||||
help=('Branch to analyze'),
|
||||
)
|
||||
parser.add_argument(
|
||||
'repos',
|
||||
nargs='*',
|
||||
help=('Repos to analyze, '
|
||||
'repo should be e.g. openstack/glance'),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
log_level = logging.ERROR
|
||||
if args.verbosity >= 3:
|
||||
log_level = logging.DEBUG
|
||||
elif args.verbosity >= 2:
|
||||
log_level = logging.INFO
|
||||
elif args.verbosity >= 1:
|
||||
log_level = logging.WARNING
|
||||
|
||||
# Set up logging, including making some loggers quiet.
|
||||
logging.basicConfig(
|
||||
format='%(levelname)7s: %(message)s',
|
||||
stream=sys.stdout,
|
||||
level=log_level,
|
||||
)
|
||||
logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING)
|
||||
|
||||
workdir = tempfile.mkdtemp(prefix='releases-')
|
||||
LOG.debug('creating temporary files in %s' % workdir)
|
||||
|
||||
def cleanup_workdir():
|
||||
if args.cleanup:
|
||||
shutil.rmtree(workdir, True)
|
||||
else:
|
||||
LOG.info('not cleaning up %s' % workdir)
|
||||
atexit.register(cleanup_workdir)
|
||||
|
||||
# Remove any inherited PAGER environment variable to avoid
|
||||
# blocking the output waiting for input.
|
||||
os.environ['PAGER'] = ''
|
||||
output = []
|
||||
|
||||
for repo in args.repos:
|
||||
current = {
|
||||
'repo': repo,
|
||||
'branch': args.branch,
|
||||
'commits': None,
|
||||
'error': False,
|
||||
'not_yet_released': False,
|
||||
'msg': ''
|
||||
}
|
||||
url = 'https://opendev.org/{}'.format(repo)
|
||||
res = requests.get(url)
|
||||
if res.status_code == 404:
|
||||
current.update({'error': True})
|
||||
current.update(
|
||||
{'msg': "fatal: repository '{}' not found".format(url)})
|
||||
output.append(current)
|
||||
continue
|
||||
|
||||
# Start by checking out master, always. We need the repo
|
||||
# checked out before we can tell if the stable branch
|
||||
# really exists.
|
||||
gitutils.clone_repo(
|
||||
workdir,
|
||||
repo,
|
||||
branch='master',
|
||||
)
|
||||
|
||||
# Set some git configuration values to allow us to perform
|
||||
# local operations like tagging.
|
||||
gitutils.ensure_basic_git_config(
|
||||
workdir, repo,
|
||||
{'user.email': 'openstack-infra@lists.openstack.org',
|
||||
'user.name': 'OpenStack Proposal Bot'},
|
||||
)
|
||||
|
||||
# Determine which branch we should actually be looking
|
||||
# at. Assume any series for which there is no stable
|
||||
# branch will be on 'master'.
|
||||
if gitutils.stable_branch_exists(workdir, repo, args.branch):
|
||||
branch = 'stable/' + args.branch
|
||||
else:
|
||||
branch = 'master'
|
||||
|
||||
if branch != 'master':
|
||||
# Check out the repo again to the right branch if we
|
||||
# didn't get it the first time.
|
||||
gitutils.clone_repo(
|
||||
workdir,
|
||||
repo,
|
||||
branch=branch,
|
||||
)
|
||||
|
||||
# look at the previous tag for the parent of the commit
|
||||
# getting the new release
|
||||
previous_tag = gitutils.get_latest_tag(workdir, repo, always=False)
|
||||
|
||||
if not previous_tag:
|
||||
current.update({'not_yet_released': True})
|
||||
current.update(
|
||||
{'msg': '{} has not yet been released'.format(repo)})
|
||||
output.append(current)
|
||||
continue
|
||||
|
||||
start_range = previous_tag
|
||||
head_sha = gitutils.get_head(workdir, repo)
|
||||
|
||||
if not start_range:
|
||||
current.update({'not_yet_released': True})
|
||||
current.update(
|
||||
{'msg': '{} has not yet been released'.format(repo)})
|
||||
output.append(current)
|
||||
continue
|
||||
|
||||
commits = git_log(workdir, repo, (start_range, head_sha),
|
||||
extra_args=[
|
||||
'--no-color',
|
||||
'--no-merges',
|
||||
'--graph',
|
||||
'--format=%h %ci %s'])
|
||||
|
||||
current.update({'commits': commits})
|
||||
output.append(current)
|
||||
|
||||
out = generate_output(output, output_format=args.format,
|
||||
ignore_no_results=args.ignore_no_results,
|
||||
ignore_errors=args.ignore_errors,
|
||||
ignore_not_yet_released=args.ignore_not_yet_released,
|
||||
ignore_all=args.ignore_all)
|
||||
print("".join(out))
|
||||
return 0
|
||||
|
|
@ -307,6 +307,20 @@ def check_ancestry(workdir, repo, old_version, sha):
|
|||
return False
|
||||
|
||||
|
||||
def get_head(workdir, repo):
|
||||
cmd = ['git', 'log', '-n', '1', '--pretty=tformat:%h']
|
||||
try:
|
||||
return processutils.check_output(
|
||||
cmd,
|
||||
cwd=os.path.join(workdir, repo),
|
||||
stderr=subprocess.STDOUT,
|
||||
).decode('utf-8').strip()
|
||||
except processutils.CalledProcessError as e:
|
||||
LOG.warning('failed to retrieve HEAD: %s [%s]',
|
||||
e, e.output.strip())
|
||||
return None
|
||||
|
||||
|
||||
def get_latest_tag(workdir, repo, sha=None, always=True):
|
||||
cmd = ['git', 'describe', '--abbrev=0']
|
||||
if always:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,156 @@
|
|||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
import textwrap
|
||||
|
||||
from oslotest import base
|
||||
|
||||
from openstack_releases.cmds import list_unreleased_changes as luc
|
||||
|
||||
|
||||
class TestStableStatus(base.BaseTestCase):
|
||||
|
||||
_data = json.loads(textwrap.dedent('''
|
||||
[
|
||||
{
|
||||
"repo": "openstack/castellan",
|
||||
"branch": "victoria",
|
||||
"commits": {
|
||||
"range": [
|
||||
"3.2.0",
|
||||
"3a3a738"
|
||||
],
|
||||
"logs": [
|
||||
"* 3a3a738 2020-06-26 13:51:12 +0200 Bump vault version"
|
||||
]
|
||||
},
|
||||
"error": false,
|
||||
"not_yet_released": false,
|
||||
"msg": ""
|
||||
},
|
||||
{
|
||||
"repo": "openstack/devstack-plugin-amqp1",
|
||||
"branch": "victoria",
|
||||
"commits": null,
|
||||
"error": false,
|
||||
"not_yet_released": true,
|
||||
"msg": "openstack/devstack-plugin-amqp1 has not yet been released"
|
||||
},
|
||||
{
|
||||
"repo": "openstack/oslo.cache",
|
||||
"branch": "victoria",
|
||||
"commits": {
|
||||
"range": [
|
||||
"2.5.0",
|
||||
"f3f006c"
|
||||
],
|
||||
"logs": [
|
||||
"* a5ff884 2020-05-04 18:18:10 +0200 Align contributing doc with oslo's policy"
|
||||
]
|
||||
},
|
||||
"error": false,
|
||||
"not_yet_released": false,
|
||||
"msg": ""
|
||||
},
|
||||
{
|
||||
"repo": "openstack/oslo.service",
|
||||
"branch": "victoria",
|
||||
"commits": {
|
||||
"range": [
|
||||
"2.3.1",
|
||||
"585768b"
|
||||
],
|
||||
"logs": []
|
||||
},
|
||||
"error": false,
|
||||
"not_yet_released": false,
|
||||
"msg": ""
|
||||
},
|
||||
{
|
||||
"repo": "openstack/devstack-plugin-kafka",
|
||||
"branch": "victoria",
|
||||
"commits": null,
|
||||
"error": false,
|
||||
"not_yet_released": true,
|
||||
"msg": "openstack/devstack-plugin-kafka has not yet been released"
|
||||
},
|
||||
{
|
||||
"repo": "openstack/tttt",
|
||||
"branch": "victoria",
|
||||
"commits": null,
|
||||
"error": true,
|
||||
"not_yet_released": false,
|
||||
"msg": "fatal: repository 'https://opendev.org/openstack/tttt' not found"
|
||||
}
|
||||
]
|
||||
''')) # noqa
|
||||
|
||||
def test_filter_results(self):
|
||||
results = luc.filter_results(self._data)
|
||||
self.assertEqual(6, len(results))
|
||||
results = luc.filter_results(self._data, ignore_all=True)
|
||||
self.assertEqual(2, len(results))
|
||||
results = luc.filter_results(self._data, ignore_errors=True)
|
||||
self.assertEqual(5, len(results))
|
||||
results = luc.filter_results(self._data, ignore_not_yet_released=True)
|
||||
self.assertEqual(4, len(results))
|
||||
results = luc.filter_results(self._data,
|
||||
ignore_not_yet_released=True,
|
||||
ignore_errors=True)
|
||||
self.assertEqual(3, len(results))
|
||||
results = luc.filter_results(self._data, ignore_no_results=True)
|
||||
self.assertEqual(2, len(results))
|
||||
|
||||
def test_generate_output(self):
|
||||
results = luc.generate_output(self._data, output_format='json')
|
||||
self.assertEqual(json.dumps(self._data, indent=4), results)
|
||||
results = luc.generate_output(self._data, ignore_all=True)
|
||||
expected_result = textwrap.dedent('''\
|
||||
[ Unreleased changes in openstack/castellan (victoria) ]
|
||||
Changes between 3.2.0 and 3a3a738
|
||||
* 3a3a738 2020-06-26 13:51:12 +0200 Bump vault version
|
||||
[ Unreleased changes in openstack/oslo.cache (victoria) ]
|
||||
Changes between 2.5.0 and f3f006c
|
||||
* a5ff884 2020-05-04 18:18:10 +0200 Align contributing doc with oslo's policy''') # noqa
|
||||
self.assertEqual(expected_result, "".join(results))
|
||||
results = luc.generate_output(self._data, output_format='yaml',
|
||||
ignore_all=True)
|
||||
expected_result = textwrap.dedent("""\
|
||||
---
|
||||
- repo: openstack/castellan
|
||||
branch: victoria
|
||||
commits:
|
||||
range:
|
||||
- 3.2.0
|
||||
- 3a3a738
|
||||
logs:
|
||||
- '* 3a3a738 2020-06-26 13:51:12 +0200 Bump vault version'
|
||||
error: false
|
||||
not_yet_released: false
|
||||
msg: ''
|
||||
- repo: openstack/oslo.cache
|
||||
branch: victoria
|
||||
commits:
|
||||
range:
|
||||
- 2.5.0
|
||||
- f3f006c
|
||||
logs:
|
||||
- "* a5ff884 2020-05-04 18:18:10 +0200 Align contributing doc\\
|
||||
\\ with oslo's policy"
|
||||
error: false
|
||||
not_yet_released: false
|
||||
msg: ''
|
||||
""") # noqa
|
||||
self.assertEqual(expected_result, results)
|
||||
|
|
@ -24,6 +24,7 @@ packages = openstack_releases
|
|||
console_scripts =
|
||||
validate-request = openstack_releases.cmds.validate:main
|
||||
list-changes = openstack_releases.cmds.list_changes:main
|
||||
list-unreleased-changes = openstack_releases.cmds.list_unreleased_changes:main
|
||||
list-constraints = openstack_releases.cmds.list_constraints:main
|
||||
new-release = openstack_releases.cmds.new_release:main
|
||||
format-yaml = openstack_releases.cmds.reformat_yaml:main
|
||||
|
|
|
|||
|
|
@ -17,46 +17,20 @@
|
|||
if [[ $# -lt 2 ]]; then
|
||||
echo "Usage: $(basename $0) <branch> <repo> [<repo>...]"
|
||||
echo "repo should be e.g. openstack/glance"
|
||||
echo
|
||||
echo "Example: $(basename $0) victoria oslo.rootwrap"
|
||||
echo "Example: $(basename $0) independent reno bugfix"
|
||||
echo
|
||||
echo "For further details about how to use the command:"
|
||||
echo "tox -e venv -- list-unreleased-changes --help"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
branch="$1"
|
||||
shift
|
||||
repos="$@"
|
||||
|
||||
TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
BASEDIR=$(dirname $TOOLSDIR)
|
||||
source $TOOLSDIR/functions
|
||||
|
||||
# Make sure no pager is configured so the output is not blocked
|
||||
export PAGER=
|
||||
|
||||
setup_temp_space 'list-unreleased'
|
||||
|
||||
function list_changes {
|
||||
title "Unreleased changes in $repo ($branch)"
|
||||
clone_repo $repo $branch
|
||||
if [[ $? -ne 0 ]]; then
|
||||
return 1
|
||||
if [[ -z "$VIRTUAL_ENV" ]]; then
|
||||
if [[ ! -d .tox/venv ]]; then
|
||||
tox -e venv --notest
|
||||
fi
|
||||
cd $repo
|
||||
prev_tag=$(get_last_tag)
|
||||
if [ -z "$prev_tag" ]; then
|
||||
echo "$repo has not yet been released"
|
||||
else
|
||||
echo
|
||||
end_sha=$(git log -n 1 --pretty=tformat:%h)
|
||||
echo "Changes between $prev_tag and $end_sha"
|
||||
echo
|
||||
git log --no-color --no-merges --format='%h %ci %s' \
|
||||
--graph ${prev_tag}..${end_sha}
|
||||
echo
|
||||
fi
|
||||
}
|
||||
source ./.tox/venv/bin/activate
|
||||
fi
|
||||
|
||||
# Show the unreleased changes for each repository.
|
||||
for repo in $repos; do
|
||||
cd $MYTMPDIR
|
||||
echo
|
||||
list_changes "$repo"
|
||||
done
|
||||
list-unreleased-changes $@
|
||||
|
|
|
|||
Loading…
Reference in New Issue