From 70be592ba5b85c27b2f473d723b6ab7d18251b5c Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 3 Oct 2017 16:21:30 -0400 Subject: [PATCH] add release notes output to the list-changes job Import the release-notes command from the release-tools repository and integrate it with the list-changes command so that we see the release notes for an upcoming release. This lets us review the release notes output for warnings (like not having a documentation page linked) and errors (such as reno failing to work). Change-Id: I710606fe44601fe5414bfc82f25894319f16e558 Signed-off-by: Doug Hellmann --- openstack_releases/cmds/list_changes.py | 40 + openstack_releases/cmds/release_notes.py | 130 +++ openstack_releases/gitutils.py | 14 + openstack_releases/release_notes.py | 422 ++++++++++ openstack_releases/rst2txt.py | 966 +++++++++++++++++++++++ requirements.txt | 6 + setup.cfg | 1 + 7 files changed, 1579 insertions(+) create mode 100644 openstack_releases/cmds/release_notes.py create mode 100644 openstack_releases/release_notes.py create mode 100644 openstack_releases/rst2txt.py diff --git a/openstack_releases/cmds/list_changes.py b/openstack_releases/cmds/list_changes.py index 5960a1e32b..f1ff68042c 100644 --- a/openstack_releases/cmds/list_changes.py +++ b/openstack_releases/cmds/list_changes.py @@ -34,6 +34,7 @@ import requests from openstack_releases import defaults from openstack_releases import gitutils from openstack_releases import governance +from openstack_releases import release_notes from openstack_releases import yamlutils @@ -503,4 +504,43 @@ def main(): git_range, extra_args=['--no-merges', '--topo-order']) + # Before we try to produce the release notes we need the + # tag to exist in the local repository. + if not tag_exists: + header('Applying Temporary Tag') + gitutils.add_tag( + workdir, + project['repo'], + new_release['version'], + project['sha'], + ) + header('Release Notes') + try: + first_release = len(deliverable_info.get('releases', [])) == 1 + notes = release_notes.generate_release_notes( + repo=project['repo'], + repo_path=os.path.join(workdir, project['repo']), + start_revision=start_range, + end_revision=new_release['version'], + show_dates=True, + skip_requirement_merges=True, + is_stable=branch.startswith('stable/'), + series=series, + email='test-job@openstack.org', + email_from='test-job@openstack.org', + email_reply_to='noreply@openstack.org', + email_tags='', + include_pypi_link=False, + changes_only=False, + first_release=first_release, + repo_name=project['repo'], + description='', + publishing_dir_name=project['repo'], + ) + except Exception as e: + logging.exception('Failed to produce release notes') + else: + print('\n') + print(notes) + return 0 diff --git a/openstack_releases/cmds/release_notes.py b/openstack_releases/cmds/release_notes.py new file mode 100644 index 0000000000..dacf4b7f9f --- /dev/null +++ b/openstack_releases/cmds/release_notes.py @@ -0,0 +1,130 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os + +from openstack_releases import release_notes + + +def main(): + parser = argparse.ArgumentParser( + prog='release_notes', + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("repo", metavar='path', action="store", + help="repository directory, for example" + " 'openstack/cliff'", + ) + parser.add_argument('repo_name', action='store', + help='The name of the repository being released', + ) + parser.add_argument("start_revision", metavar='revision', + action="store", + help="start revision, for example '1.8.0'", + ) + parser.add_argument("end_revision", metavar='revision', + action="store", + nargs='?', + help="end revision, for example '1.9.0'" + " (default: HEAD)", + default="HEAD") + parser.add_argument('--changes-only', + action='store_true', + default=False, + help='List only the change summary, without details', + ) + parser.add_argument('--include-pypi-link', + action='store_true', + default=False, + help='include a pypi hyperlink for the library', + ) + parser.add_argument('--first-release', + action='store_true', + default=False, + help='this is the first release of the project', + ) + parser.add_argument("--skip-requirement-merges", + action='store_true', default=False, + help="skip requirement update commit messages" + " (default: False)") + parser.add_argument("--show-dates", + action='store_true', default=False, + help="show dates in the change log") + parser.add_argument("--series", "-s", + default="", + help="release series name, such as 'kilo'", + ) + parser.add_argument("--stable", + default=False, + action='store_true', + help="this is a stable release", + ) + parser.add_argument('--description', + action='store', + help=('A brief description for the repository being ' + 'released'), + ) + parser.add_argument('--publishing-dir-name', + action='store', + help=('The directory on tarballs.openstack.org ' + 'and docs.openstack.org containing the ' + 'published artifacts for this package'), + ) + + email_group = parser.add_argument_group('email settings') + email_group.add_argument( + "--email", "-e", + action='store_true', default=False, + help="output a fully formed email message", + ) + email_group.add_argument( + "--email-reply-to", + default="openstack-dev@lists.openstack.org", + help="follow-up for discussions, defaults to %(default)s", + ) + email_group.add_argument( + "--email-from", "--from", + default=os.environ.get('EMAIL', ''), + help="source of the email, defaults to $EMAIL", + ) + email_group.add_argument( + "--email-tags", + default="", + help="extra topic tags for email subject, e.g. '[oslo]'", + ) + args = parser.parse_args() + + repo_path = os.path.abspath(args.repo) + + notes = release_notes.generate_release_notes( + repo=args.repo, + repo_path=repo_path, + start_revision=args.start_revision, + end_revision=args.end_revision, + show_dates=args.show_dates, + skip_requirement_merges=args.skip_requirement_merges, + is_stable=args.stable, + series=args.series, + email=args.email, + email_from=args.email_from, + email_reply_to=args.email_reply_to, + email_tags=args.email_tags, + include_pypi_link=args.include_pypi_link, + changes_only=args.changes_only, + first_release=args.first_release, + repo_name=args.repo_name, + description=args.description, + publishing_dir_name=args.publishing_dir_name or args.repo_name, + ) + print(notes) + return 0 diff --git a/openstack_releases/gitutils.py b/openstack_releases/gitutils.py index b225529745..bbca82a903 100644 --- a/openstack_releases/gitutils.py +++ b/openstack_releases/gitutils.py @@ -224,6 +224,20 @@ def get_latest_tag(workdir, repo, sha=None): return None +def add_tag(workdir, repo, tag, sha): + cmd = ['git', 'tag', '-m', 'temporary tag', tag, sha] + try: + return subprocess.check_output( + cmd, + cwd=os.path.join(workdir, repo), + stderr=subprocess.STDOUT, + ).decode('utf-8').strip() + except subprocess.CalledProcessError as e: + LOG.warning('failed to add tag: %s [%s]', + e, e.output.strip()) + return None + + def get_branches(workdir, repo): try: output = subprocess.check_output( diff --git a/openstack_releases/release_notes.py b/openstack_releases/release_notes.py new file mode 100644 index 0000000000..95fe5c264c --- /dev/null +++ b/openstack_releases/release_notes.py @@ -0,0 +1,422 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Generates a standard set of release notes for a repository.""" + +import glob +import logging +import os +import random +import subprocess + +import jinja2 +import parawrap +from reno import config as reno_config +from reno import formatter +from reno import loader + +from openstack_releases import rst2txt + +LOG = logging.getLogger(__name__) + + +EMOTIONS = [ + 'are amped to', + 'are chuffed to', + 'contentedly', + 'are delighted to', + 'eagerly', + 'are ecstatic to', + 'enthusiastically', + 'are excited to', + 'exuberantly', + 'are glad to', + 'are gleeful to', + 'are happy to', + 'high-spiritedly', + 'are jazzed to', + 'joyfully', + 'jubilantly', + 'are overjoyed to', + 'are pleased to', + 'are psyched to', + 'are pumped to', + 'are satisfied to', + 'are stoked to', + 'are thrilled to', + 'are tickled pink to', +] + +# The email headers for generating a message to go right into sendmail +# or msmtp. +EMAIL_HEADER_TPL = """ +{%- if email %} +From: {{email_from}} +To: {{email_to}} +Reply-To: {{email_reply_to}} +Subject: {{email_tags}} {{project}} {{end_rev}}{% if series %} ({{series}}){% endif %} +{% endif %} +""" + +PYPI_URL_TPL = 'https://pypi.python.org/pypi/%s' + +# This will be replaced with template values and then wrapped using parawrap +# to correctly wrap at paragraph boundaries... + +HEADER_RELEASE_TPL = """ +We {{ emotion }} announce the release of: + +{% if description %} +{{ project }} {{ end_rev }}: {{ description }} +{% else %} +{{ project }} {{ end_rev }} +{% endif %} + +{% if first_release -%} +This is the first release of {{project}}. +{%- endif %} +{% if series -%} +This release is part of the {{series}} {% if stable_series %}stable {% endif %}release series. +{%- endif %} +{% if source_url %} + +The source is available from: + + {{ source_url }} +{% endif %} + +Download the package from: + +{% if pypi_url %} + {{ pypi_url }} +{% else %} + https://tarballs.openstack.org/{{publishing_dir_name}}/ +{% endif %} +{% if bug_url %} + +Please report issues through launchpad: + + {{ bug_url }} +{% endif %} + +For more details, please see below. +""" + +# This will just be replaced with template values (no wrapping applied). +CHANGE_RELEASE_TPL = """{% if reno_notes %}{{ reno_notes }}{% endif %} +{% if changes %}{{ change_header }}{% if skip_requirement_merges %} + +NOTE: Skipping requirement commits... +{%- endif %} + +{% for change in changes -%} +{{ change }} +{% endfor %} +{%- endif %} +{% if diff_stats %} +{% if not first_release -%} +Diffstat (except docs and test files) +------------------------------------- + +{% for change in diff_stats -%} +{{ change }} +{% endfor %} +{%- endif %} +{% if requirement_changes %} +Requirements updates +-------------------- + +{% for change in requirement_changes -%} +{{ change }} +{% endfor %} +{%- endif %} +{% endif %} +""" + +CHANGES_ONLY_TPL = """{{ change_header }} +{% for change in changes -%} +{{ change }} +{% endfor %} +""" + +RELEASE_CANDIDATE_TPL = """ +Hello everyone, + +A new release candidate for {{project}} for the end of the {{series|capitalize}} +cycle is available! You can find the source code tarball at: + + https://tarballs.openstack.org/{{publishing_dir_name}}/ + +Unless release-critical issues are found that warrant a release +candidate respin, this candidate will be formally released as the +final {{series|capitalize}} release. You are therefore strongly +encouraged to test and validate this tarball! + +Alternatively, you can directly test the stable/{{series|lower}} release +branch at: + + http://git.openstack.org/cgit/openstack/{{project}}/log/?h=stable/{{series|lower}} + +Release notes for {{project}} can be found at: + + http://docs.openstack.org/releasenotes/{{project}}/ + +{% if bug_url -%} +If you find an issue that could be considered release-critical, please +file it at: + + {{bug_url}} + +and tag it *{{series|lower}}-rc-potential* to bring it to the {{project}} +release crew's attention. +{%- endif %} +""" + + +def parse_readme(repo_path): + sections = { + 'bug_url': '', + 'source_url': '', + } + readme_formats = ['rst', 'md'] + for k in readme_formats: + readme_path = os.path.join(repo_path, 'README.%s' % k) + try: + f = open(readme_path, 'r', encoding='utf-8') + f.close() + break + except IOError: + continue + else: + LOG.warning("No README file found in %s\n" + % repo_path) + return sections + + with open(readme_path, 'r', encoding='utf-8') as fh: + for line in fh: + for (name, key_name) in [("Bugs:", "bug_url"), + ("Source:", 'source_url')]: + pieces = line.split(name, 1) + if len(pieces) == 2: + candidate = pieces[1].strip() + if 'http' in candidate: + sections[key_name] = candidate + for (k, v) in sections.items(): + if not v: + what = k.replace("_", " ") + LOG.warning("No %s found in '%s'\n" + % (what, readme_path)) + return sections + + +def expand_template(contents, params): + if not params: + params = {} + tpl = jinja2.Template(source=contents, undefined=jinja2.StrictUndefined) + return tpl.render(**params) + + +def run_cmd(cmd, cwd=None, encoding='utf-8'): + # Created since currently the 'processutils' function doesn't take a + # working directory, which we need in this example due to the different + # working directories we run programs in... + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=cwd) + stdout, stderr = p.communicate() + if p.returncode != 0: + raise subprocess.ProcessExecutionError(stdout=stdout, + stderr=stderr, + exit_code=p.returncode, + cmd=cmd) + return stdout.decode(encoding), stderr.decode(encoding) + + +def is_skippable_commit(skip_requirement_merges, line): + return (skip_requirement_merges and + line.lower().endswith('updated from global requirements')) + + +def generate_release_notes(repo, repo_path, + start_revision, end_revision, + show_dates, skip_requirement_merges, + is_stable, series, + email, email_from, + email_reply_to, email_tags, + include_pypi_link, + changes_only, + first_release, + repo_name, description, + publishing_dir_name, + ): + """Return the text of the release notes. + + :param repo: The name of the repo. + :param repo_path: Path to the repo repository on disk. + :param start_revision: First reference for finding change log. + :param end_revision: Final reference for finding change log. + :param show_dates: Boolean indicating whether or not to show dates + in the output. + :param skip_requirement_merges: Boolean indicating whether to + skip merge commits for requirements changes. + :param is_stable: Boolean indicating whether this is a stable + series or not. + :param series: String holding the name of the series. + :param email: Boolean indicating whether the output format should + be an email message. + :param email_from: String containing the sender email address. + :param email_reply_to: String containing the email reply-to address. + :param email_tags: String containing the email header topic tags to add. + :param include_pypi_link: Boolean indicating whether or not to + include an automatically generated link to the PyPI package + page. + :param changes_only: Boolean indicating whether to limit output to + the list of changes, without any extra data. + :param first_release: Boolean indicating whether this is the first + release of the project + :param repo_name: Name of the repo + :param description: Description of the repo + :param publishing_dir_name: The directory on publishings.openstack.org + containing the package. + + """ + # Determine if this is a release candidate or not. + is_release_candidate = 'rc' in end_revision + + # Do not mention the series in independent model since there is none + if series == 'independent': + series = '' + + if not email_from: + raise RuntimeError('No email-from specified') + + # Get the commits that are in the desired range... + git_range = "%s..%s" % (start_revision, end_revision) + if show_dates: + format = "--format=%h %ci %s" + else: + format = "--oneline" + cmd = ["git", "log", "--no-color", format, "--no-merges", git_range] + stdout, stderr = run_cmd(cmd, cwd=repo_path) + changes = [] + for commit_line in stdout.splitlines(): + commit_line = commit_line.strip() + if not commit_line or is_skippable_commit(skip_requirement_merges, + commit_line): + continue + else: + changes.append(commit_line) + + # Filter out any requirement file changes... + requirement_changes = [] + requirement_files = list(glob.glob(os.path.join(repo_path, + '*requirements*.txt'))) + if requirement_files: + cmd = ['git', 'diff', '-U0', '--no-color', git_range] + cmd.extend(requirement_files) + stdout, stderr = run_cmd(cmd, cwd=repo_path) + requirement_changes = [line.strip() + for line in stdout.splitlines() if line.strip()] + + # Get statistics about the range given... + cmd = ['git', 'diff', '--stat', '--no-color', git_range] + stdout, stderr = run_cmd(cmd, cwd=repo_path) + diff_stats = [] + for line in stdout.splitlines(): + line = line.strip() + if not line or line.find("tests") != -1 or line.startswith("doc"): + continue + diff_stats.append(line) + + # Extract + valdiate needed sections from readme... + readme_sections = parse_readme(repo_path) + change_header = ["Changes in %s %s" % (repo_name, git_range)] + change_header.append("-" * len(change_header[0])) + + # Look for reno notes for this version. + if not changes_only: + cfg = reno_config.Config( + reporoot=repo_path, + ) + branch = None + if is_stable and series: + branch = 'origin/stable/%s' % series + cfg.override(branch=branch) + ldr = loader.Loader(conf=cfg) + if end_revision in ldr.versions: + rst_notes = formatter.format_report( + loader=ldr, + config=cfg, + versions_to_include=[end_revision], + ) + reno_notes = rst2txt.convert(rst_notes).decode('utf-8') + else: + LOG.warning( + ('Did not find revision %r in list of versions ' + 'with release notes %r, skipping reno'), + end_revision, ldr.versions, + ) + reno_notes = '' + else: + reno_notes = '' + + # The recipient for announcements should always be the + # release-announce@lists.openstack.org ML (except for + # release-test and release candidates) + email_to = 'release-announce@lists.openstack.org' + if repo == 'openstack-release-test': + email_to = 'release-job-failures@lists.openstack.org' + elif is_release_candidate: + email_to = 'openstack-dev@lists.openstack.org' + + params = dict(readme_sections) + params.update({ + 'project': repo_name, + 'description': description, + 'end_rev': end_revision, + 'range': git_range, + 'lib': repo_path, + 'skip_requirement_merges': skip_requirement_merges, + 'changes': changes, + 'requirement_changes': requirement_changes, + 'diff_stats': diff_stats, + 'change_header': "\n".join(change_header), + 'emotion': random.choice(EMOTIONS), + 'stable_series': is_stable, + 'series': series, + 'email': email, + 'email_from': email_from, + 'email_to': email_to, + 'email_reply_to': email_reply_to, + 'email_tags': email_tags, + 'reno_notes': reno_notes, + 'first_release': first_release, + 'publishing_dir_name': publishing_dir_name, + }) + if include_pypi_link: + params['pypi_url'] = PYPI_URL_TPL % repo_name + else: + params['pypi_url'] = None + + response = [] + if changes_only: + response.append(expand_template(CHANGES_ONLY_TPL, params)) + else: + if email: + email_header = expand_template(EMAIL_HEADER_TPL.strip(), params) + response.append(email_header.lstrip()) + if is_release_candidate: + response.append(expand_template(RELEASE_CANDIDATE_TPL, params)) + else: + header = expand_template(HEADER_RELEASE_TPL.strip(), params) + response.append(parawrap.fill(header)) + response.append(expand_template(CHANGE_RELEASE_TPL, params)) + return '\n'.join(response) diff --git a/openstack_releases/rst2txt.py b/openstack_releases/rst2txt.py new file mode 100644 index 0000000000..abebc7234a --- /dev/null +++ b/openstack_releases/rst2txt.py @@ -0,0 +1,966 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Convert reStructuredText to plain text. + +Most of this code is adapted from sphinx.writers.text, Copyright +2007-2015 by the Sphinx team (see +git://github.com/sphinx-doc/sphinx). The code is imported here because +it has to be modified to not rely on other internal Sphinx components +that we don't have set up, such as a Builder and Application object. + +""" + +import itertools +import os +import re +import textwrap + +from docutils import core +from docutils import nodes +from docutils.utils import column_width +from docutils import writers +from six.moves import zip_longest + +from sphinx import addnodes + + +def convert(source): + """Given reStructuredText input, return formatted plain text output.""" + return core.publish_string( + source=source, + writer=TextWriter(), + ) + + +class TextWrapper(textwrap.TextWrapper): + """Custom subclass that uses a different word separator regex.""" + + wordsep_re = re.compile( + r'(\s+|' # any whitespace + r'(?<=\s)(?::[a-z-]+:)?`\S+|' # interpreted text start + r'[^\s\w]*\w+[a-zA-Z]-(?=\w+[a-zA-Z])|' # hyphenated words + r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))') # em-dash + + def _wrap_chunks(self, chunks): + """_wrap_chunks(chunks : [string]) -> [string] + + The original _wrap_chunks uses len() to calculate width. + This method respects wide/fullwidth characters for width adjustment. + """ + drop_whitespace = getattr(self, 'drop_whitespace', True) # py25 compat + lines = [] + if self.width <= 0: + raise ValueError("invalid width %r (must be > 0)" % self.width) + + chunks.reverse() + + while chunks: + cur_line = [] + cur_len = 0 + + if lines: + indent = self.subsequent_indent + else: + indent = self.initial_indent + + width = self.width - column_width(indent) + + if drop_whitespace and chunks[-1].strip() == '' and lines: + del chunks[-1] + + while chunks: + l = column_width(chunks[-1]) + + if cur_len + l <= width: + cur_line.append(chunks.pop()) + cur_len += l + + else: + break + + if chunks and column_width(chunks[-1]) > width: + self._handle_long_word(chunks, cur_line, cur_len, width) + + if drop_whitespace and cur_line and cur_line[-1].strip() == '': + del cur_line[-1] + + if cur_line: + lines.append(indent + ''.join(cur_line)) + + return lines + + def _break_word(self, word, space_left): + """_break_word(word : string, space_left : int) -> (string, string) + + Break line by unicode width instead of len(word). + """ + total = 0 + for i, c in enumerate(word): + total += column_width(c) + if total > space_left: + return word[:i - 1], word[i - 1:] + return word, '' + + def _split(self, text): + """_split(text : string) -> [string] + + Override original method that only split by 'wordsep_re'. + This '_split' split wide-characters into chunk by one character. + """ + def split(t): + return textwrap.TextWrapper._split(self, t) + chunks = [] + for chunk in split(text): + for w, g in itertools.groupby(chunk, column_width): + if w == 1: + chunks.extend(split(''.join(g))) + else: + chunks.extend(list(g)) + return chunks + + def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): + """Deal with long words + + _handle_long_word(chunks : [string], + cur_line : [string], + cur_len : int, width : int) + + Override original method for using self._break_word() instead + of slice. + + """ + space_left = max(width - cur_len, 1) + if self.break_long_words: + l, r = self._break_word(reversed_chunks[-1], space_left) + cur_line.append(l) + reversed_chunks[-1] = r + + elif not cur_line: + cur_line.append(reversed_chunks.pop()) + + +MAXWIDTH = 70 +STDINDENT = 3 + + +def my_wrap(text, width=MAXWIDTH, **kwargs): + w = TextWrapper(width=width, **kwargs) + return w.wrap(text) + + +class TextWriter(writers.Writer): + supported = ('text',) + settings_spec = ('No options here.', '', ()) + settings_defaults = {} + + output = None + + def __init__(self): + writers.Writer.__init__(self) + self.translator_class = TextTranslator + + def translate(self): + visitor = self.translator_class(self.document) + self.document.walkabout(visitor) + self.output = visitor.body + + +class TextTranslator(nodes.NodeVisitor): + sectionchars = '*=-~"+`' + + def __init__(self, document): + nodes.NodeVisitor.__init__(self, document) + + self.nl = os.linesep + self.states = [[]] + self.stateindent = [0] + self.list_counter = [] + self.sectionlevel = 0 + self.lineblocklevel = 0 + self.table = None + + def add_text(self, text): + self.states[-1].append((-1, text)) + + def new_state(self, indent=STDINDENT): + self.states.append([]) + self.stateindent.append(indent) + + def end_state(self, wrap=True, end=[''], first=None): + content = self.states.pop() + maxindent = sum(self.stateindent) + indent = self.stateindent.pop() + result = [] + toformat = [] + + def do_format(): + if not toformat: + return + if wrap: + res = my_wrap(''.join(toformat), width=MAXWIDTH - maxindent) + else: + res = ''.join(toformat).splitlines() + if end: + res += end + result.append((indent, res)) + for itemindent, item in content: + if itemindent == -1: + toformat.append(item) + else: + do_format() + result.append((indent + itemindent, item)) + toformat = [] + do_format() + if first is not None and result: + itemindent, item = result[0] + result_rest, result = result[1:], [] + if item: + toformat = [first + ' '.join(item)] + do_format() # re-create `result` from `toformat` + _dummy, new_item = result[0] + result.insert(0, (itemindent - indent, [new_item[0]])) + result[1] = (itemindent, new_item[1:]) + result.extend(result_rest) + self.states[-1].extend(result) + + def visit_document(self, node): + self.new_state(0) + + def depart_document(self, node): + self.end_state() + self.body = self.nl.join(line and (' ' * indent + line) + for indent, lines in self.states[0] + for line in lines) + # XXX header/footer? + + def visit_highlightlang(self, node): + raise nodes.SkipNode + + def visit_section(self, node): + self._title_char = self.sectionchars[self.sectionlevel] + self.sectionlevel += 1 + + def depart_section(self, node): + self.sectionlevel -= 1 + + def visit_topic(self, node): + self.new_state(0) + + def depart_topic(self, node): + self.end_state() + + visit_sidebar = visit_topic + depart_sidebar = depart_topic + + def visit_rubric(self, node): + self.new_state(0) + self.add_text('-[ ') + + def depart_rubric(self, node): + self.add_text(' ]-') + self.end_state() + + def visit_compound(self, node): + pass + + def depart_compound(self, node): + pass + + def visit_glossary(self, node): + pass + + def depart_glossary(self, node): + pass + + def visit_title(self, node): + if isinstance(node.parent, nodes.Admonition): + self.add_text(node.astext() + ': ') + raise nodes.SkipNode + self.new_state(0) + + def depart_title(self, node): + if isinstance(node.parent, nodes.section): + char = self._title_char + else: + char = '^' + text = ''.join(x[1] for x in self.states.pop() if x[0] == -1) + self.stateindent.pop() + self.states[-1].append( + (0, ['', text, '%s' % (char * column_width(text)), ''])) + + def visit_subtitle(self, node): + pass + + def depart_subtitle(self, node): + pass + + def visit_attribution(self, node): + self.add_text('-- ') + + def depart_attribution(self, node): + pass + + def visit_desc(self, node): + pass + + def depart_desc(self, node): + pass + + def visit_desc_signature(self, node): + self.new_state(0) + if node.parent['objtype'] in ('class', 'exception'): + self.add_text('%s ' % node.parent['objtype']) + + def depart_desc_signature(self, node): + # XXX: wrap signatures in a way that makes sense + self.end_state(wrap=False, end=None) + + def visit_desc_name(self, node): + pass + + def depart_desc_name(self, node): + pass + + def visit_desc_addname(self, node): + pass + + def depart_desc_addname(self, node): + pass + + def visit_desc_type(self, node): + pass + + def depart_desc_type(self, node): + pass + + def visit_desc_returns(self, node): + self.add_text(' -> ') + + def depart_desc_returns(self, node): + pass + + def visit_desc_parameterlist(self, node): + self.add_text('(') + self.first_param = 1 + + def depart_desc_parameterlist(self, node): + self.add_text(')') + + def visit_desc_parameter(self, node): + if not self.first_param: + self.add_text(', ') + else: + self.first_param = 0 + self.add_text(node.astext()) + raise nodes.SkipNode + + def visit_desc_optional(self, node): + self.add_text('[') + + def depart_desc_optional(self, node): + self.add_text(']') + + def visit_desc_annotation(self, node): + pass + + def depart_desc_annotation(self, node): + pass + + def visit_desc_content(self, node): + self.new_state() + self.add_text(self.nl) + + def depart_desc_content(self, node): + self.end_state() + + def visit_figure(self, node): + self.new_state() + + def depart_figure(self, node): + self.end_state() + + def visit_caption(self, node): + pass + + def depart_caption(self, node): + pass + + def visit_productionlist(self, node): + self.new_state() + names = [] + for production in node: + names.append(production['tokenname']) + maxlen = max(len(name) for name in names) + lastname = None + for production in node: + if production['tokenname']: + self.add_text(production['tokenname'].ljust(maxlen) + ' ::=') + lastname = production['tokenname'] + elif lastname is not None: + self.add_text('%s ' % (' ' * len(lastname))) + self.add_text(production.astext() + self.nl) + self.end_state(wrap=False) + raise nodes.SkipNode + + def visit_footnote(self, node): + self._footnote = node.children[0].astext().strip() + self.new_state(len(self._footnote) + 3) + + def depart_footnote(self, node): + self.end_state(first='[%s] ' % self._footnote) + + def visit_citation(self, node): + if len(node) and isinstance(node[0], nodes.label): + self._citlabel = node[0].astext() + else: + self._citlabel = '' + self.new_state(len(self._citlabel) + 3) + + def depart_citation(self, node): + self.end_state(first='[%s] ' % self._citlabel) + + def visit_label(self, node): + raise nodes.SkipNode + + def visit_legend(self, node): + pass + + def depart_legend(self, node): + pass + + # XXX: option list could use some better styling + + def visit_option_list(self, node): + pass + + def depart_option_list(self, node): + pass + + def visit_option_list_item(self, node): + self.new_state(0) + + def depart_option_list_item(self, node): + self.end_state() + + def visit_option_group(self, node): + self._firstoption = True + + def depart_option_group(self, node): + self.add_text(' ') + + def visit_option(self, node): + if self._firstoption: + self._firstoption = False + else: + self.add_text(', ') + + def depart_option(self, node): + pass + + def visit_option_string(self, node): + pass + + def depart_option_string(self, node): + pass + + def visit_option_argument(self, node): + self.add_text(node['delimiter']) + + def depart_option_argument(self, node): + pass + + def visit_description(self, node): + pass + + def depart_description(self, node): + pass + + def visit_tabular_col_spec(self, node): + raise nodes.SkipNode + + def visit_colspec(self, node): + self.table[0].append(node['colwidth']) + raise nodes.SkipNode + + def visit_tgroup(self, node): + pass + + def depart_tgroup(self, node): + pass + + def visit_thead(self, node): + pass + + def depart_thead(self, node): + pass + + def visit_tbody(self, node): + self.table.append('sep') + + def depart_tbody(self, node): + pass + + def visit_row(self, node): + self.table.append([]) + + def depart_row(self, node): + pass + + def visit_entry(self, node): + if 'morerows' in node or 'morecols' in node: + raise NotImplementedError('Column or row spanning cells are ' + 'not implemented.') + self.new_state(0) + + def depart_entry(self, node): + text = self.nl.join(self.nl.join(x[1]) for x in self.states.pop()) + self.stateindent.pop() + self.table[-1].append(text) + + def visit_table(self, node): + if self.table: + raise NotImplementedError('Nested tables are not supported.') + self.new_state(0) + self.table = [[]] + + def depart_table(self, node): + lines = self.table[1:] + fmted_rows = [] + colwidths = self.table[0] + realwidths = colwidths[:] + separator = 0 + # don't allow paragraphs in table cells for now + for line in lines: + if line == 'sep': + separator = len(fmted_rows) + else: + cells = [] + for i, cell in enumerate(line): + par = my_wrap(cell, width=colwidths[i]) + if par: + maxwidth = max(column_width(x) for x in par) + else: + maxwidth = 0 + realwidths[i] = max(realwidths[i], maxwidth) + cells.append(par) + fmted_rows.append(cells) + + def writesep(char='-'): + out = ['+'] + for width in realwidths: + out.append(char * (width + 2)) + out.append('+') + self.add_text(''.join(out) + self.nl) + + def writerow(row): + lines = zip_longest(*row) + for line in lines: + out = ['|'] + for i, cell in enumerate(line): + if cell: + adjust_len = len(cell) - column_width(cell) + out.append(' ' + cell.ljust( + realwidths[i] + 1 + adjust_len)) + else: + out.append(' ' * (realwidths[i] + 2)) + out.append('|') + self.add_text(''.join(out) + self.nl) + + for i, row in enumerate(fmted_rows): + if separator and i == separator: + writesep('=') + else: + writesep('-') + writerow(row) + writesep('-') + self.table = None + self.end_state(wrap=False) + + def visit_acks(self, node): + self.new_state(0) + self.add_text( + ', '.join(n.astext() for n in node.children[0].children) + + '.') + self.end_state() + raise nodes.SkipNode + + def visit_image(self, node): + if 'alt' in node.attributes: + self.add_text('[image: %s]' % node['alt']) + self.add_text('[image]') + raise nodes.SkipNode + + def visit_transition(self, node): + indent = sum(self.stateindent) + self.new_state(0) + self.add_text('=' * (MAXWIDTH - indent)) + self.end_state() + raise nodes.SkipNode + + def visit_bullet_list(self, node): + self.list_counter.append(-1) + + def depart_bullet_list(self, node): + self.list_counter.pop() + + def visit_enumerated_list(self, node): + self.list_counter.append(node.get('start', 1) - 1) + + def depart_enumerated_list(self, node): + self.list_counter.pop() + + def visit_definition_list(self, node): + self.list_counter.append(-2) + + def depart_definition_list(self, node): + self.list_counter.pop() + + def visit_list_item(self, node): + if self.list_counter[-1] == -1: + # bullet list + self.new_state(2) + elif self.list_counter[-1] == -2: + # definition list + pass + else: + # enumerated list + self.list_counter[-1] += 1 + self.new_state(len(str(self.list_counter[-1])) + 2) + + def depart_list_item(self, node): + if self.list_counter[-1] == -1: + self.end_state(first='* ') + elif self.list_counter[-1] == -2: + pass + else: + self.end_state(first='%s. ' % self.list_counter[-1]) + + def visit_definition_list_item(self, node): + self._li_has_classifier = len(node) >= 2 and \ + isinstance(node[1], nodes.classifier) + + def depart_definition_list_item(self, node): + pass + + def visit_term(self, node): + self.new_state(0) + + def depart_term(self, node): + if not self._li_has_classifier: + self.end_state(end=None) + + def visit_termsep(self, node): + self.add_text(', ') + raise nodes.SkipNode + + def visit_classifier(self, node): + self.add_text(' : ') + + def depart_classifier(self, node): + self.end_state(end=None) + + def visit_definition(self, node): + self.new_state() + + def depart_definition(self, node): + self.end_state() + + def visit_field_list(self, node): + pass + + def depart_field_list(self, node): + pass + + def visit_field(self, node): + pass + + def depart_field(self, node): + pass + + def visit_field_name(self, node): + self.new_state(0) + + def depart_field_name(self, node): + self.add_text(':') + self.end_state(end=None) + + def visit_field_body(self, node): + self.new_state() + + def depart_field_body(self, node): + self.end_state() + + def visit_centered(self, node): + pass + + def depart_centered(self, node): + pass + + def visit_hlist(self, node): + pass + + def depart_hlist(self, node): + pass + + def visit_hlistcol(self, node): + pass + + def depart_hlistcol(self, node): + pass + + def visit_admonition(self, node): + self.new_state(0) + + def depart_admonition(self, node): + self.end_state() + + def _visit_admonition(self, node): + self.new_state(2) + + def _make_depart_admonition(name): + def depart_admonition(self, node): + self.end_state(first=name.title() + ': ') + return depart_admonition + + visit_attention = _visit_admonition + depart_attention = _make_depart_admonition('attention') + visit_caution = _visit_admonition + depart_caution = _make_depart_admonition('caution') + visit_danger = _visit_admonition + depart_danger = _make_depart_admonition('danger') + visit_error = _visit_admonition + depart_error = _make_depart_admonition('error') + visit_hint = _visit_admonition + depart_hint = _make_depart_admonition('hint') + visit_important = _visit_admonition + depart_important = _make_depart_admonition('important') + visit_note = _visit_admonition + depart_note = _make_depart_admonition('note') + visit_tip = _visit_admonition + depart_tip = _make_depart_admonition('tip') + visit_warning = _visit_admonition + depart_warning = _make_depart_admonition('warning') + visit_seealso = _visit_admonition + depart_seealso = _make_depart_admonition('seealso') + + def visit_versionmodified(self, node): + self.new_state(0) + + def depart_versionmodified(self, node): + self.end_state() + + def visit_literal_block(self, node): + self.new_state() + + def depart_literal_block(self, node): + self.end_state(wrap=False) + + def visit_doctest_block(self, node): + self.new_state(0) + + def depart_doctest_block(self, node): + self.end_state(wrap=False) + + def visit_line_block(self, node): + self.new_state() + self.lineblocklevel += 1 + + def depart_line_block(self, node): + self.lineblocklevel -= 1 + self.end_state(wrap=False, end=None) + if not self.lineblocklevel: + self.add_text('\n') + + def visit_line(self, node): + pass + + def depart_line(self, node): + self.add_text('\n') + + def visit_block_quote(self, node): + self.new_state() + + def depart_block_quote(self, node): + self.end_state() + + def visit_compact_paragraph(self, node): + pass + + def depart_compact_paragraph(self, node): + pass + + def visit_paragraph(self, node): + if not isinstance(node.parent, nodes.Admonition) or \ + isinstance(node.parent, addnodes.seealso): + self.new_state(0) + + def depart_paragraph(self, node): + if not isinstance(node.parent, nodes.Admonition) or \ + isinstance(node.parent, addnodes.seealso): + self.end_state() + + def visit_target(self, node): + pass + + def depart_target(self, node): + self.add_text(' (%s)' % node['refuri']) + + def visit_index(self, node): + raise nodes.SkipNode + + def visit_toctree(self, node): + raise nodes.SkipNode + + def visit_substitution_definition(self, node): + raise nodes.SkipNode + + def visit_pending_xref(self, node): + pass + + def depart_pending_xref(self, node): + pass + + def visit_reference(self, node): + pass + + def depart_reference(self, node): + pass + + def visit_number_reference(self, node): + text = nodes.Text(node.get('title', '#')) + self.visit_Text(text) + raise nodes.SkipNode + + def visit_download_reference(self, node): + pass + + def depart_download_reference(self, node): + pass + + def visit_emphasis(self, node): + self.add_text('*') + + def depart_emphasis(self, node): + self.add_text('*') + + def visit_literal_emphasis(self, node): + self.add_text('*') + + def depart_literal_emphasis(self, node): + self.add_text('*') + + def visit_strong(self, node): + self.add_text('**') + + def depart_strong(self, node): + self.add_text('**') + + def visit_literal_strong(self, node): + self.add_text('**') + + def depart_literal_strong(self, node): + self.add_text('**') + + def visit_abbreviation(self, node): + self.add_text('') + + def depart_abbreviation(self, node): + if node.hasattr('explanation'): + self.add_text(' (%s)' % node['explanation']) + + def visit_title_reference(self, node): + self.add_text('*') + + def depart_title_reference(self, node): + self.add_text('*') + + def visit_literal(self, node): + self.add_text('"') + + def depart_literal(self, node): + self.add_text('"') + + def visit_subscript(self, node): + self.add_text('_') + + def depart_subscript(self, node): + pass + + def visit_superscript(self, node): + self.add_text('^') + + def depart_superscript(self, node): + pass + + def visit_footnote_reference(self, node): + self.add_text('[%s]' % node.astext()) + raise nodes.SkipNode + + def visit_citation_reference(self, node): + self.add_text('[%s]' % node.astext()) + raise nodes.SkipNode + + def visit_Text(self, node): + self.add_text(node.astext()) + + def depart_Text(self, node): + pass + + def visit_generated(self, node): + pass + + def depart_generated(self, node): + pass + + def visit_inline(self, node): + if 'xref' in node['classes'] or 'term' in node['classes']: + self.add_text('*') + + def depart_inline(self, node): + if 'xref' in node['classes'] or 'term' in node['classes']: + self.add_text('*') + + def visit_container(self, node): + pass + + def depart_container(self, node): + pass + + def visit_problematic(self, node): + pass + + def depart_problematic(self, node): + pass + + def visit_system_message(self, node): + raise nodes.SkipNode + + def visit_comment(self, node): + raise nodes.SkipNode + + def visit_meta(self, node): + # only valid for HTML + raise nodes.SkipNode + + def visit_raw(self, node): + if 'text' in node.get('format', '').split(): + self.body.append(node.astext()) + raise nodes.SkipNode + + def visit_math(self, node): + raise nodes.SkipNode + + visit_math_block = visit_math + + def unknown_visit(self, node): + raise NotImplementedError('Unknown node: ' + node.__class__.__name__) diff --git a/requirements.txt b/requirements.txt index 8092996262..28e94fb905 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,9 @@ tqdm packaging>=15.2 mwclient==0.8.1 jsonschema>=2.6.0 + +# For release notes generation. +Jinja2>=2.6 # BSD License (3 clause) +parawrap +reno>=2.0.0 +sphinx>=1.6.2 # BSD diff --git a/setup.cfg b/setup.cfg index 3a3f01a18f..1af97cbc2b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,6 +39,7 @@ console_scripts = propose-library-branches = openstack_releases.cmds.propose_library_branches:main edit-deliverable = openstack_releases.cmds.edit_deliverable:main send-mail = openstack_releases.cmds.mail:main + release-notes = openstack_releases.cmds.release_notes:main [extras] sphinxext =