582 lines
21 KiB
Python
582 lines
21 KiB
Python
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import argparse
|
|
import atexit
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
|
|
from openstack_releases.cmds.interactive_release import clean_changes
|
|
from openstack_releases.cmds.interactive_release import yes_no_prompt
|
|
from openstack_releases import gitutils
|
|
from openstack_releases import series_status
|
|
from openstack_releases import yamlutils
|
|
|
|
# Release models that support release candidates.
|
|
_USES_RCS = ['cycle-with-milestones', 'cycle-trailing', 'cycle-with-rc']
|
|
|
|
LOG = logging.getLogger('')
|
|
|
|
|
|
def get_deliverable_data(series, deliverable):
|
|
deliverable_filename = 'deliverables/%s/%s.yaml' % (
|
|
series, deliverable)
|
|
with open(deliverable_filename, 'r', encoding='utf-8') as f:
|
|
return yamlutils.loads(f)
|
|
|
|
|
|
def increment_version(old_version, increment):
|
|
"""Compute the new version based on the previous value.
|
|
|
|
:param old_version: Parts of the version string for the last
|
|
release.
|
|
:type old_version: list(str)
|
|
:param increment: Which positions to increment.
|
|
:type increment: tuple(int)
|
|
"""
|
|
new_version_parts = []
|
|
clear = False
|
|
for cur, inc in zip(old_version, increment):
|
|
if clear:
|
|
new_version_parts.append('0')
|
|
else:
|
|
new_version_parts.append(str(int(cur) + inc))
|
|
if inc:
|
|
clear = True
|
|
return new_version_parts
|
|
|
|
|
|
def increment_milestone_version(old_version, release_type):
|
|
"""Increment a version using the rules for milestone projects.
|
|
|
|
:param old_version: Parts of the version string for the last
|
|
release.
|
|
:type old_version: list(str)
|
|
:param release_type: Either ``'milestone'`` or ``'rc'``.
|
|
:type release_type: str
|
|
"""
|
|
if release_type == 'milestone':
|
|
if 'b' in old_version[-1]:
|
|
# Not the first milestone
|
|
new_version_parts = old_version[:-1]
|
|
next_milestone = int(old_version[-1][2:]) + 1
|
|
new_version_parts.append('0b{}'.format(next_milestone))
|
|
else:
|
|
new_version_parts = increment_version(old_version, (1, 0, 0))
|
|
new_version_parts.append('0b1')
|
|
elif release_type == 'rc':
|
|
new_version_parts = old_version[:-1]
|
|
if 'b' in old_version[-1]:
|
|
# First RC
|
|
new_version_parts.append('0rc1')
|
|
elif 'rc' in old_version[-1]:
|
|
# A deliverable exists so we can handle normally
|
|
next_rc = int(old_version[-1][3:]) + 1
|
|
new_version_parts.append('0rc{}'.format(next_rc))
|
|
else:
|
|
# No milestone or rc exists, fallback to the same logic
|
|
# as for '0b1' and increment the major version.
|
|
new_version_parts = increment_version(old_version, (1, 0, 0))
|
|
new_version_parts.append('0rc1')
|
|
else:
|
|
raise ValueError('Unknown release type {!r}'.format(release_type))
|
|
return new_version_parts
|
|
|
|
|
|
def get_last_series_info(series, deliverable):
|
|
all_series = sorted(os.listdir('deliverables'))
|
|
prev_series = all_series[all_series.index(series) - 1]
|
|
try:
|
|
return get_deliverable_data(prev_series, deliverable)
|
|
except (IOError, OSError, KeyError) as e:
|
|
raise RuntimeError(
|
|
'Could not determine previous version: %s' % (e,))
|
|
|
|
|
|
def feature_increment(last_release):
|
|
"""Increment the major release number.
|
|
|
|
How much do we need to increment the feature number to provision
|
|
for future stable releases in skipped series, based on last release
|
|
found.
|
|
"""
|
|
return max(1, last_release['depth'])
|
|
|
|
|
|
def get_release_history(series, deliverable):
|
|
"""Retrieve the history of releases for a given deliverable.
|
|
|
|
Returns an array of arrays containing the releases for each series,
|
|
in reverse chronological order starting from specified series.
|
|
"""
|
|
if series == '_independent':
|
|
included_series = ['_independent']
|
|
else:
|
|
status = series_status.SeriesStatus.default()
|
|
all_series = list(status.names)
|
|
LOG.debug('all series %s', all_series)
|
|
included_series = all_series[all_series.index(series):]
|
|
release_history = []
|
|
LOG.debug('building release history')
|
|
for current_series in included_series:
|
|
try:
|
|
deliv_info = get_deliverable_data(current_series, deliverable)
|
|
releases = deliv_info['releases'] or []
|
|
except (IOError, OSError, KeyError):
|
|
releases = []
|
|
LOG.debug('%s releases: %s', current_series,
|
|
[r['version'] for r in releases])
|
|
release_history.append(releases)
|
|
return release_history
|
|
|
|
|
|
def get_last_release(release_history, deliverable, release_type):
|
|
depth = 0
|
|
for releases in release_history:
|
|
LOG.debug('looking for previous version in %s',
|
|
[r['version'] for r in releases])
|
|
if releases:
|
|
LOG.debug('using %s', releases[-1]['version'])
|
|
return dict({'depth': depth}, **releases[-1])
|
|
elif release_type == 'bugfix':
|
|
raise RuntimeError(
|
|
'The first release for a series must '
|
|
'be at least a feature release to allow '
|
|
'for stable releases from the previous series.')
|
|
depth = depth + 1
|
|
|
|
return None
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
'series',
|
|
help='the name of the release series to scan',
|
|
)
|
|
parser.add_argument(
|
|
'deliverable',
|
|
help='the base name of the deliverable file',
|
|
)
|
|
parser.add_argument(
|
|
'-v', '--verbose',
|
|
default=False,
|
|
action='store_true',
|
|
help='be more chatty',
|
|
)
|
|
parser.add_argument(
|
|
'-i', '--interactive',
|
|
default=False,
|
|
action='store_true',
|
|
help='Be interactive and only make releases when instructed'
|
|
)
|
|
parser.add_argument(
|
|
'release_type',
|
|
choices=('bugfix', 'feature', 'major', 'milestone', 'rc',
|
|
'procedural', 'eol', 'em', 'releasefix'),
|
|
help='the type of release to generate',
|
|
)
|
|
parser.add_argument(
|
|
'--no-cleanup',
|
|
dest='cleanup',
|
|
default=True,
|
|
action='store_false',
|
|
help='do not remove temporary files',
|
|
)
|
|
parser.add_argument(
|
|
'--force',
|
|
default=False,
|
|
action='store_true',
|
|
help=('force a new tag, even if the HEAD of the '
|
|
'branch is already tagged'),
|
|
)
|
|
parser.add_argument(
|
|
'--debug',
|
|
default=False,
|
|
action='store_true',
|
|
help='show tracebacks on errors',
|
|
)
|
|
parser.add_argument(
|
|
'--stable-branch',
|
|
default=False,
|
|
action='store_true',
|
|
help='create a new stable branch from the release',
|
|
)
|
|
parser.add_argument(
|
|
'--intermediate-branch',
|
|
default=False,
|
|
action='store_true',
|
|
help='create a new intermediate branch with the form bugfix/n.n; '
|
|
'this is usually needed only by projects with a '
|
|
'cycle-with-intermediary release model'
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
# Set up logging, including making some loggers quiet.
|
|
logging.basicConfig(
|
|
format='%(levelname)7s: %(message)s',
|
|
stream=sys.stdout,
|
|
level=logging.DEBUG if args.verbose else logging.INFO,
|
|
)
|
|
logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING)
|
|
|
|
is_procedural = args.release_type in 'procedural'
|
|
is_retagging = is_procedural or args.release_type == 'releasefix'
|
|
is_eol = args.release_type == 'eol'
|
|
is_em = args.release_type == 'em'
|
|
force_tag = args.force
|
|
|
|
workdir = tempfile.mkdtemp(prefix='releases-')
|
|
LOG.info('creating temporary files in %s', workdir)
|
|
|
|
def error(msg):
|
|
if args.debug:
|
|
raise msg
|
|
else:
|
|
parser.error(msg)
|
|
|
|
def cleanup_workdir():
|
|
if args.cleanup:
|
|
shutil.rmtree(workdir, True)
|
|
else:
|
|
LOG.warning('not cleaning up %s', workdir)
|
|
atexit.register(cleanup_workdir)
|
|
|
|
# Allow for independent projects.
|
|
series = args.series
|
|
if series.lstrip('_') == 'independent':
|
|
series = '_independent'
|
|
|
|
# Load existing deliverable data.
|
|
try:
|
|
deliverable_info = get_deliverable_data(
|
|
series, args.deliverable)
|
|
except (IOError, OSError) as e:
|
|
error(e)
|
|
|
|
# Ensure we have a list for releases, even if it is empty.
|
|
if deliverable_info.get('releases') is None:
|
|
deliverable_info['releases'] = []
|
|
|
|
try:
|
|
release_history = get_release_history(series, args.deliverable)
|
|
this_series_history = release_history[0]
|
|
last_release = get_last_release(
|
|
release_history,
|
|
args.deliverable,
|
|
args.release_type,
|
|
)
|
|
except RuntimeError as err:
|
|
error(err)
|
|
if last_release:
|
|
last_version = last_release['version'].split('.')
|
|
else:
|
|
last_version = None
|
|
LOG.debug('last_version %r', last_version)
|
|
diff_start = None
|
|
|
|
add_stable_branch = args.stable_branch or is_procedural
|
|
|
|
add_intermediate_branch = args.intermediate_branch
|
|
|
|
# Validate new tag can be applied
|
|
if last_version and 'eol' in last_version[0]:
|
|
raise ValueError('Cannot create new release after EOL tagging.')
|
|
|
|
if last_version is None:
|
|
# Deliverables that have never been released before should
|
|
# start at 0.1.0, indicating they are not feature complete or
|
|
# stable but have features.
|
|
LOG.debug('defaulting to 0.1.0 for first release')
|
|
new_version_parts = ['0', '1', '0']
|
|
|
|
elif args.release_type in ('milestone', 'rc'):
|
|
force_tag = True
|
|
if deliverable_info['release-model'] not in _USES_RCS:
|
|
raise ValueError('Cannot compute RC for {} project {}'.format(
|
|
deliverable_info['release-model'], args.deliverable))
|
|
new_version_parts = increment_milestone_version(
|
|
last_version, args.release_type)
|
|
LOG.debug('computed new version %s release type %s',
|
|
new_version_parts, args.release_type)
|
|
# We are going to take some special steps for the first
|
|
# release candidate, so figure out if that is what this
|
|
# release will be.
|
|
if args.release_type == 'rc' and new_version_parts[-1][3:] == '1':
|
|
add_stable_branch = True
|
|
|
|
elif args.release_type == 'procedural':
|
|
# NOTE(dhellmann): We always compute the new version based on
|
|
# the highest version on the branch, rather than the branch
|
|
# base. If the differences are only patch levels the results
|
|
# do not change, but if there was a minor version update then
|
|
# the new version needs to be incremented based on that.
|
|
new_version_parts = increment_version(last_version, (
|
|
0, feature_increment(last_release), 0)
|
|
)
|
|
|
|
# NOTE(dhellmann): Save the SHAs for the commits where the
|
|
# branch was created in each repo, even though that is
|
|
# unlikely to be the same as the last_version, because commits
|
|
# further down the stable branch will not be in the history of
|
|
# the master branch and so we can't tag them as part of the
|
|
# new series *AND* we always want stable branches created from
|
|
# master.
|
|
prev_info = get_last_series_info(series, args.deliverable)
|
|
for b in prev_info['branches']:
|
|
if b['name'].startswith('stable/'):
|
|
last_branch_base = b['location'].split('.')
|
|
break
|
|
else:
|
|
raise ValueError(
|
|
'Could not find a version in branch before {}'.format(
|
|
series)
|
|
)
|
|
if last_version != last_branch_base:
|
|
LOG.warning('last_version %s branch base %s',
|
|
'.'.join(last_version), '.'.join(last_branch_base))
|
|
for r in prev_info['releases']:
|
|
if r['version'] == '.'.join(last_branch_base):
|
|
last_version_hashes = {
|
|
p['repo']: p['hash']
|
|
for p in r['projects']
|
|
}
|
|
break
|
|
else:
|
|
raise ValueError(
|
|
('Could not find SHAs for tag '
|
|
'{} in old deliverable file').format(
|
|
'.'.join(last_version))
|
|
)
|
|
|
|
elif args.release_type == 'releasefix':
|
|
increment = (0, 0, 1)
|
|
new_version_parts = increment_version(last_version, increment)
|
|
last_version_hashes = {
|
|
p['repo']: p['hash']
|
|
for p in last_release['projects']
|
|
}
|
|
# Go back 2 releases so the release announcement includes the
|
|
# actual changes.
|
|
try:
|
|
diff_start_release = this_series_history[-2]
|
|
except IndexError:
|
|
# We do not have 2 releases in this series yet, so go back
|
|
# to the stable branch creation point.
|
|
prev_info = get_last_series_info(series, args.deliverable)
|
|
for b in prev_info['branches']:
|
|
if b['name'].startswith('stable/'):
|
|
diff_start = b['location']
|
|
LOG.info('using branch point from previous '
|
|
'series as diff-start: %r', diff_start)
|
|
break
|
|
else:
|
|
diff_start = diff_start_release['version']
|
|
LOG.info('using release from same series as diff-start: %r',
|
|
diff_start)
|
|
|
|
elif is_eol or is_em:
|
|
last_version_hashes = {
|
|
p['repo']: p['hash']
|
|
for p in last_release['projects']
|
|
}
|
|
increment = None
|
|
new_version_parts = None
|
|
new_version = '{}-{}'.format(args.series, args.release_type)
|
|
|
|
else:
|
|
increment = {
|
|
'bugfix': (0, 0, 1),
|
|
'feature': (0, feature_increment(last_release), 0),
|
|
'major': (1, 0, 0),
|
|
}[args.release_type]
|
|
new_version_parts = increment_version(last_version, increment)
|
|
LOG.debug('computed new version %s', new_version_parts)
|
|
|
|
if new_version_parts is not None:
|
|
# The EOL/EM tag version string is computed above and the parts
|
|
# list is set to None to avoid recomputing it here.
|
|
new_version = '.'.join(new_version_parts)
|
|
|
|
if 'releases' not in deliverable_info:
|
|
deliverable_info['releases'] = []
|
|
|
|
LOG.info('going from %s to %s', last_version, new_version)
|
|
|
|
projects = []
|
|
changes = 0
|
|
for repo in deliverable_info['repository-settings'].keys():
|
|
LOG.info('processing %s', repo)
|
|
|
|
# Look for the most recent time the repo was tagged and use
|
|
# that info as the old sha.
|
|
previous_sha = None
|
|
previous_tag = None
|
|
found = False
|
|
for release in reversed(deliverable_info['releases']):
|
|
for project in release['projects']:
|
|
if project['repo'] == repo:
|
|
previous_sha = project.get('hash')
|
|
previous_tag = release['version']
|
|
LOG.info('last tagged as %s at %s',
|
|
previous_tag, previous_sha)
|
|
found = True
|
|
break
|
|
if found:
|
|
break
|
|
|
|
if is_retagging or (
|
|
is_em and deliverable_info['release-model'] != 'untagged'):
|
|
# Always use the last tagged hash, which should be coming
|
|
# from the previous series or last release.
|
|
sha = last_version_hashes[repo]
|
|
|
|
else:
|
|
# Figure out the hash for the HEAD of the branch.
|
|
gitutils.clone_repo(workdir, repo)
|
|
|
|
branches = gitutils.get_branches(workdir, repo)
|
|
version = 'origin/stable/%s' % series
|
|
if not any(branch for branch in branches
|
|
if branch.endswith(version)):
|
|
version = 'master'
|
|
|
|
sha = gitutils.sha_for_tag(workdir, repo, version)
|
|
|
|
# Check out the working repo to the sha
|
|
gitutils.checkout_ref(workdir, repo, sha)
|
|
|
|
if is_retagging:
|
|
changes += 1
|
|
LOG.info('re-tagging %s at %s (%s)', repo, sha, previous_tag)
|
|
if is_procedural:
|
|
comment = 'procedural tag to support creating stable branch'
|
|
else:
|
|
comment = 'procedural tag to handle release job failure'
|
|
new_project = {
|
|
'repo': repo,
|
|
'hash': sha,
|
|
'comment': comment,
|
|
}
|
|
projects.append(new_project)
|
|
|
|
elif is_eol or is_em:
|
|
changes += 1
|
|
LOG.info('tagging %s %s at %s',
|
|
repo,
|
|
args.release_type.upper(),
|
|
sha)
|
|
new_project = {
|
|
'repo': repo,
|
|
'hash': sha,
|
|
}
|
|
projects.append(new_project)
|
|
|
|
elif previous_sha != sha or force_tag:
|
|
# TODO(tonyb): Do this early and also prompt for release type.
|
|
# Once we do that we can probably deprecate interactive-release
|
|
if args.interactive:
|
|
# NOTE(tonyb): This is pretty much just copied from
|
|
# interactive-release
|
|
last_tag = '.'.join(last_version)
|
|
change_lines = list(clean_changes(gitutils.changes_since(
|
|
workdir, repo, last_tag).splitlines()))
|
|
max_changes_show = 100
|
|
LOG.info('')
|
|
if last_tag:
|
|
LOG.info("%s changes to %s since %s are:",
|
|
len(change_lines), repo, last_tag)
|
|
else:
|
|
LOG.info("%s changes to %s are:", len(change_lines), repo)
|
|
for sha, descr in change_lines[0:max_changes_show]:
|
|
LOG.info("* %s %s", sha[:7], descr)
|
|
leftover_change_lines = change_lines[max_changes_show:]
|
|
if leftover_change_lines:
|
|
LOG.info(" and %s more changes...",
|
|
len(leftover_change_lines))
|
|
LOG.info('')
|
|
|
|
changes += 1
|
|
LOG.info('advancing %s from %s (%s) to %s',
|
|
repo, previous_sha, previous_tag, sha)
|
|
new_project = {
|
|
'repo': repo,
|
|
'hash': sha,
|
|
}
|
|
projects.append(new_project)
|
|
|
|
else:
|
|
LOG.info('%s already tagged at most recent commit, skipping', repo)
|
|
|
|
new_release_info = {
|
|
'version': new_version,
|
|
'projects': projects,
|
|
}
|
|
if diff_start:
|
|
new_release_info['diff-start'] = diff_start
|
|
deliverable_info['releases'].append(new_release_info)
|
|
|
|
if add_stable_branch:
|
|
branch_name = 'stable/{}'.format(series)
|
|
|
|
# First check if this branch is already defined
|
|
if 'branches' in deliverable_info:
|
|
for branch in deliverable_info['branches']:
|
|
if branch.get('name') == branch_name:
|
|
LOG.debug('Branch %s already exists, skipping',
|
|
branch_name)
|
|
add_stable_branch = False
|
|
break
|
|
|
|
if add_stable_branch:
|
|
LOG.info('adding stable branch at %s', new_version)
|
|
deliverable_info.setdefault('branches', []).append({
|
|
'name': branch_name,
|
|
'location': new_version,
|
|
})
|
|
|
|
if add_intermediate_branch:
|
|
new_branch = new_version.rsplit('.', 1)[0]
|
|
branch_name = 'bugfix/{}'.format(new_branch)
|
|
|
|
# First check if this branch is already defined
|
|
if 'branches' in deliverable_info:
|
|
for branch in deliverable_info['branches']:
|
|
if branch.get('name') == branch_name:
|
|
LOG.debug('Branch %s already exists, skipping',
|
|
branch_name)
|
|
add_intermediate_branch = False
|
|
break
|
|
|
|
if add_intermediate_branch:
|
|
LOG.info('adding intermediate branch at %s', new_version)
|
|
deliverable_info.setdefault('branches', []).append({
|
|
'name': branch_name,
|
|
'location': new_version,
|
|
})
|
|
|
|
create_release = changes > 0
|
|
if create_release and args.interactive:
|
|
create_release = yes_no_prompt(
|
|
'Create a release in %s containing those changes? ' % series)
|
|
|
|
if create_release:
|
|
deliverable_filename = 'deliverables/%s/%s.yaml' % (
|
|
series, args.deliverable)
|
|
with open(deliverable_filename, 'w', encoding='utf-8') as f:
|
|
f.write(yamlutils.dumps(deliverable_info))
|