add release notes output to the list-changes job

Import the release-notes command from the release-tools repository and
integrate it with the list-changes command so that we see the release
notes for an upcoming release. This lets us review the release notes
output for warnings (like not having a documentation page linked) and
errors (such as reno failing to work).

Change-Id: I710606fe44601fe5414bfc82f25894319f16e558
Signed-off-by: Doug Hellmann <doug@doughellmann.com>

openstack_releases/cmds/list_changes.py

@@ -34,6 +34,7 @@ import requests
 from openstack_releases import defaults
 from openstack_releases import gitutils
 from openstack_releases import governance
+from openstack_releases import release_notes
 from openstack_releases import yamlutils
 
 
@@ -503,4 +504,43 @@ def main():
                     git_range,
                     extra_args=['--no-merges', '--topo-order'])
 
+            # Before we try to produce the release notes we need the
+            # tag to exist in the local repository.
+            if not tag_exists:
+                header('Applying Temporary Tag')
+                gitutils.add_tag(
+                    workdir,
+                    project['repo'],
+                    new_release['version'],
+                    project['sha'],
+                )
+            header('Release Notes')
+            try:
+                first_release = len(deliverable_info.get('releases', [])) == 1
+                notes = release_notes.generate_release_notes(
+                    repo=project['repo'],
+                    repo_path=os.path.join(workdir, project['repo']),
+                    start_revision=start_range,
+                    end_revision=new_release['version'],
+                    show_dates=True,
+                    skip_requirement_merges=True,
+                    is_stable=branch.startswith('stable/'),
+                    series=series,
+                    email='test-job@openstack.org',
+                    email_from='test-job@openstack.org',
+                    email_reply_to='noreply@openstack.org',
+                    email_tags='',
+                    include_pypi_link=False,
+                    changes_only=False,
+                    first_release=first_release,
+                    repo_name=project['repo'],
+                    description='',
+                    publishing_dir_name=project['repo'],
+                )
+            except Exception as e:
+                logging.exception('Failed to produce release notes')
+            else:
+                print('\n')
+                print(notes)
+
     return 0

openstack_releases/cmds/release_notes.py

@@ -0,0 +1,130 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import argparse
+import os
+
+from openstack_releases import release_notes
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog='release_notes',
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("repo", metavar='path', action="store",
+                        help="repository directory, for example"
+                             " 'openstack/cliff'",
+                        )
+    parser.add_argument('repo_name', action='store',
+                        help='The name of the repository being released',
+                        )
+    parser.add_argument("start_revision", metavar='revision',
+                        action="store",
+                        help="start revision, for example '1.8.0'",
+                        )
+    parser.add_argument("end_revision", metavar='revision',
+                        action="store",
+                        nargs='?',
+                        help="end revision, for example '1.9.0'"
+                             " (default: HEAD)",
+                        default="HEAD")
+    parser.add_argument('--changes-only',
+                        action='store_true',
+                        default=False,
+                        help='List only the change summary, without details',
+                        )
+    parser.add_argument('--include-pypi-link',
+                        action='store_true',
+                        default=False,
+                        help='include a pypi hyperlink for the library',
+                        )
+    parser.add_argument('--first-release',
+                        action='store_true',
+                        default=False,
+                        help='this is the first release of the project',
+                        )
+    parser.add_argument("--skip-requirement-merges",
+                        action='store_true', default=False,
+                        help="skip requirement update commit messages"
+                             " (default: False)")
+    parser.add_argument("--show-dates",
+                        action='store_true', default=False,
+                        help="show dates in the change log")
+    parser.add_argument("--series", "-s",
+                        default="",
+                        help="release series name, such as 'kilo'",
+                        )
+    parser.add_argument("--stable",
+                        default=False,
+                        action='store_true',
+                        help="this is a stable release",
+                        )
+    parser.add_argument('--description',
+                        action='store',
+                        help=('A brief description for the repository being '
+                              'released'),
+                        )
+    parser.add_argument('--publishing-dir-name',
+                        action='store',
+                        help=('The directory on tarballs.openstack.org '
+                              'and docs.openstack.org containing the '
+                              'published artifacts for this package'),
+                        )
+
+    email_group = parser.add_argument_group('email settings')
+    email_group.add_argument(
+        "--email", "-e",
+        action='store_true', default=False,
+        help="output a fully formed email message",
+    )
+    email_group.add_argument(
+        "--email-reply-to",
+        default="openstack-dev@lists.openstack.org",
+        help="follow-up for discussions, defaults to %(default)s",
+    )
+    email_group.add_argument(
+        "--email-from", "--from",
+        default=os.environ.get('EMAIL', ''),
+        help="source of the email, defaults to $EMAIL",
+    )
+    email_group.add_argument(
+        "--email-tags",
+        default="",
+        help="extra topic tags for email subject, e.g. '[oslo]'",
+    )
+    args = parser.parse_args()
+
+    repo_path = os.path.abspath(args.repo)
+
+    notes = release_notes.generate_release_notes(
+        repo=args.repo,
+        repo_path=repo_path,
+        start_revision=args.start_revision,
+        end_revision=args.end_revision,
+        show_dates=args.show_dates,
+        skip_requirement_merges=args.skip_requirement_merges,
+        is_stable=args.stable,
+        series=args.series,
+        email=args.email,
+        email_from=args.email_from,
+        email_reply_to=args.email_reply_to,
+        email_tags=args.email_tags,
+        include_pypi_link=args.include_pypi_link,
+        changes_only=args.changes_only,
+        first_release=args.first_release,
+        repo_name=args.repo_name,
+        description=args.description,
+        publishing_dir_name=args.publishing_dir_name or args.repo_name,
+    )
+    print(notes)
+    return 0

openstack_releases/gitutils.py

@@ -224,6 +224,20 @@ def get_latest_tag(workdir, repo, sha=None):
         return None
 
 
+def add_tag(workdir, repo, tag, sha):
+    cmd = ['git', 'tag', '-m', 'temporary tag', tag, sha]
+    try:
+        return subprocess.check_output(
+            cmd,
+            cwd=os.path.join(workdir, repo),
+            stderr=subprocess.STDOUT,
+        ).decode('utf-8').strip()
+    except subprocess.CalledProcessError as e:
+        LOG.warning('failed to add tag: %s [%s]',
+                    e, e.output.strip())
+        return None
+
+
 def get_branches(workdir, repo):
     try:
         output = subprocess.check_output(

openstack_releases/release_notes.py

@@ -0,0 +1,422 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Generates a standard set of release notes for a repository."""
+
+import glob
+import logging
+import os
+import random
+import subprocess
+
+import jinja2
+import parawrap
+from reno import config as reno_config
+from reno import formatter
+from reno import loader
+
+from openstack_releases import rst2txt
+
+LOG = logging.getLogger(__name__)
+
+
+EMOTIONS = [
+    'are amped to',
+    'are chuffed to',
+    'contentedly',
+    'are delighted to',
+    'eagerly',
+    'are ecstatic to',
+    'enthusiastically',
+    'are excited to',
+    'exuberantly',
+    'are glad to',
+    'are gleeful to',
+    'are happy to',
+    'high-spiritedly',
+    'are jazzed to',
+    'joyfully',
+    'jubilantly',
+    'are overjoyed to',
+    'are pleased to',
+    'are psyched to',
+    'are pumped to',
+    'are satisfied to',
+    'are stoked to',
+    'are thrilled to',
+    'are tickled pink to',
+]
+
+# The email headers for generating a message to go right into sendmail
+# or msmtp.
+EMAIL_HEADER_TPL = """
+{%- if email %}
+From: {{email_from}}
+To: {{email_to}}
+Reply-To: {{email_reply_to}}
+Subject: {{email_tags}} {{project}} {{end_rev}}{% if series %} ({{series}}){% endif %}
+{% endif %}
+"""
+
+PYPI_URL_TPL = 'https://pypi.python.org/pypi/%s'
+
+# This will be replaced with template values and then wrapped using parawrap
+# to correctly wrap at paragraph boundaries...
+
+HEADER_RELEASE_TPL = """
+We {{ emotion }} announce the release of:
+
+{% if description %}
+{{ project }} {{ end_rev }}: {{ description }}
+{% else %}
+{{ project }} {{ end_rev }}
+{% endif %}
+
+{% if first_release -%}
+This is the first release of {{project}}.
+{%- endif %}
+{% if series -%}
+This release is part of the {{series}} {% if stable_series %}stable {% endif %}release series.
+{%- endif %}
+{% if source_url %}
+
+The source is available from:
+
+    {{ source_url }}
+{% endif %}
+
+Download the package from:
+
+{% if pypi_url %}
+    {{ pypi_url }}
+{% else %}
+    https://tarballs.openstack.org/{{publishing_dir_name}}/
+{% endif %}
+{% if bug_url %}
+
+Please report issues through launchpad:
+
+    {{ bug_url }}
+{% endif %}
+
+For more details, please see below.
+"""
+
+# This will just be replaced with template values (no wrapping applied).
+CHANGE_RELEASE_TPL = """{% if reno_notes %}{{ reno_notes }}{% endif %}
+{% if changes %}{{ change_header }}{% if skip_requirement_merges %}
+
+NOTE: Skipping requirement commits...
+{%- endif %}
+
+{% for change in changes -%}
+{{ change }}
+{% endfor %}
+{%- endif %}
+{% if diff_stats %}
+{% if not first_release -%}
+Diffstat (except docs and test files)
+-------------------------------------
+
+{% for change in diff_stats -%}
+{{ change }}
+{% endfor %}
+{%- endif %}
+{% if requirement_changes %}
+Requirements updates
+--------------------
+
+{% for change in requirement_changes -%}
+{{ change }}
+{% endfor %}
+{%- endif %}
+{% endif %}
+"""
+
+CHANGES_ONLY_TPL = """{{ change_header }}
+{% for change in changes -%}
+{{ change }}
+{% endfor %}
+"""
+
+RELEASE_CANDIDATE_TPL = """
+Hello everyone,
+
+A new release candidate for {{project}} for the end of the {{series|capitalize}}
+cycle is available!  You can find the source code tarball at:
+
+    https://tarballs.openstack.org/{{publishing_dir_name}}/
+
+Unless release-critical issues are found that warrant a release
+candidate respin, this candidate will be formally released as the
+final {{series|capitalize}} release. You are therefore strongly
+encouraged to test and validate this tarball!
+
+Alternatively, you can directly test the stable/{{series|lower}} release
+branch at:
+
+    http://git.openstack.org/cgit/openstack/{{project}}/log/?h=stable/{{series|lower}}
+
+Release notes for {{project}} can be found at:
+
+    http://docs.openstack.org/releasenotes/{{project}}/
+
+{% if bug_url -%}
+If you find an issue that could be considered release-critical, please
+file it at:
+
+    {{bug_url}}
+
+and tag it *{{series|lower}}-rc-potential* to bring it to the {{project}}
+release crew's attention.
+{%- endif %}
+"""
+
+
+def parse_readme(repo_path):
+    sections = {
+        'bug_url': '',
+        'source_url': '',
+    }
+    readme_formats = ['rst', 'md']
+    for k in readme_formats:
+        readme_path = os.path.join(repo_path, 'README.%s' % k)
+        try:
+            f = open(readme_path, 'r', encoding='utf-8')
+            f.close()
+            break
+        except IOError:
+            continue
+    else:
+        LOG.warning("No README file found in %s\n"
+                    % repo_path)
+        return sections
+
+    with open(readme_path, 'r', encoding='utf-8') as fh:
+        for line in fh:
+            for (name, key_name) in [("Bugs:", "bug_url"),
+                                     ("Source:", 'source_url')]:
+                pieces = line.split(name, 1)
+                if len(pieces) == 2:
+                    candidate = pieces[1].strip()
+                    if 'http' in candidate:
+                        sections[key_name] = candidate
+    for (k, v) in sections.items():
+        if not v:
+            what = k.replace("_", " ")
+            LOG.warning("No %s found in '%s'\n"
+                        % (what, readme_path))
+    return sections
+
+
+def expand_template(contents, params):
+    if not params:
+        params = {}
+    tpl = jinja2.Template(source=contents, undefined=jinja2.StrictUndefined)
+    return tpl.render(**params)
+
+
+def run_cmd(cmd, cwd=None, encoding='utf-8'):
+    # Created since currently the 'processutils' function doesn't take a
+    # working directory, which we need in this example due to the different
+    # working directories we run programs in...
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                         cwd=cwd)
+    stdout, stderr = p.communicate()
+    if p.returncode != 0:
+        raise subprocess.ProcessExecutionError(stdout=stdout,
+                                               stderr=stderr,
+                                               exit_code=p.returncode,
+                                               cmd=cmd)
+    return stdout.decode(encoding), stderr.decode(encoding)
+
+
+def is_skippable_commit(skip_requirement_merges, line):
+    return (skip_requirement_merges and
+            line.lower().endswith('updated from global requirements'))
+
+
+def generate_release_notes(repo, repo_path,
+                           start_revision, end_revision,
+                           show_dates, skip_requirement_merges,
+                           is_stable, series,
+                           email, email_from,
+                           email_reply_to, email_tags,
+                           include_pypi_link,
+                           changes_only,
+                           first_release,
+                           repo_name, description,
+                           publishing_dir_name,
+                           ):
+    """Return the text of the release notes.
+
+    :param repo: The name of the repo.
+    :param repo_path: Path to the repo repository on disk.
+    :param start_revision: First reference for finding change log.
+    :param end_revision: Final reference for finding change log.
+    :param show_dates: Boolean indicating whether or not to show dates
+        in the output.
+    :param skip_requirement_merges: Boolean indicating whether to
+        skip merge commits for requirements changes.
+    :param is_stable: Boolean indicating whether this is a stable
+        series or not.
+    :param series: String holding the name of the series.
+    :param email: Boolean indicating whether the output format should
+        be an email message.
+    :param email_from: String containing the sender email address.
+    :param email_reply_to: String containing the email reply-to address.
+    :param email_tags: String containing the email header topic tags to add.
+    :param include_pypi_link: Boolean indicating whether or not to
+        include an automatically generated link to the PyPI package
+        page.
+    :param changes_only: Boolean indicating whether to limit output to
+        the list of changes, without any extra data.
+    :param first_release: Boolean indicating whether this is the first
+        release of the project
+    :param repo_name: Name of the repo
+    :param description: Description of the repo
+    :param publishing_dir_name: The directory on publishings.openstack.org
+        containing the package.
+
+    """
+    # Determine if this is a release candidate or not.
+    is_release_candidate = 'rc' in end_revision
+
+    # Do not mention the series in independent model since there is none
+    if series == 'independent':
+        series = ''
+
+    if not email_from:
+        raise RuntimeError('No email-from specified')
+
+    # Get the commits that are in the desired range...
+    git_range = "%s..%s" % (start_revision, end_revision)
+    if show_dates:
+        format = "--format=%h %ci %s"
+    else:
+        format = "--oneline"
+    cmd = ["git", "log", "--no-color", format, "--no-merges", git_range]
+    stdout, stderr = run_cmd(cmd, cwd=repo_path)
+    changes = []
+    for commit_line in stdout.splitlines():
+        commit_line = commit_line.strip()
+        if not commit_line or is_skippable_commit(skip_requirement_merges,
+                                                  commit_line):
+            continue
+        else:
+            changes.append(commit_line)
+
+    # Filter out any requirement file changes...
+    requirement_changes = []
+    requirement_files = list(glob.glob(os.path.join(repo_path,
+                                                    '*requirements*.txt')))
+    if requirement_files:
+        cmd = ['git', 'diff', '-U0', '--no-color', git_range]
+        cmd.extend(requirement_files)
+        stdout, stderr = run_cmd(cmd, cwd=repo_path)
+        requirement_changes = [line.strip()
+                               for line in stdout.splitlines() if line.strip()]
+
+    # Get statistics about the range given...
+    cmd = ['git', 'diff', '--stat', '--no-color', git_range]
+    stdout, stderr = run_cmd(cmd, cwd=repo_path)
+    diff_stats = []
+    for line in stdout.splitlines():
+        line = line.strip()
+        if not line or line.find("tests") != -1 or line.startswith("doc"):
+            continue
+        diff_stats.append(line)
+
+    # Extract + valdiate needed sections from readme...
+    readme_sections = parse_readme(repo_path)
+    change_header = ["Changes in %s %s" % (repo_name, git_range)]
+    change_header.append("-" * len(change_header[0]))
+
+    # Look for reno notes for this version.
+    if not changes_only:
+        cfg = reno_config.Config(
+            reporoot=repo_path,
+        )
+        branch = None
+        if is_stable and series:
+            branch = 'origin/stable/%s' % series
+        cfg.override(branch=branch)
+        ldr = loader.Loader(conf=cfg)
+        if end_revision in ldr.versions:
+            rst_notes = formatter.format_report(
+                loader=ldr,
+                config=cfg,
+                versions_to_include=[end_revision],
+            )
+            reno_notes = rst2txt.convert(rst_notes).decode('utf-8')
+        else:
+            LOG.warning(
+                ('Did not find revision %r in list of versions '
+                 'with release notes %r, skipping reno'),
+                end_revision, ldr.versions,
+            )
+            reno_notes = ''
+    else:
+        reno_notes = ''
+
+    # The recipient for announcements should always be the
+    # release-announce@lists.openstack.org ML (except for
+    # release-test and release candidates)
+    email_to = 'release-announce@lists.openstack.org'
+    if repo == 'openstack-release-test':
+        email_to = 'release-job-failures@lists.openstack.org'
+    elif is_release_candidate:
+        email_to = 'openstack-dev@lists.openstack.org'
+
+    params = dict(readme_sections)
+    params.update({
+        'project': repo_name,
+        'description': description,
+        'end_rev': end_revision,
+        'range': git_range,
+        'lib': repo_path,
+        'skip_requirement_merges': skip_requirement_merges,
+        'changes': changes,
+        'requirement_changes': requirement_changes,
+        'diff_stats': diff_stats,
+        'change_header': "\n".join(change_header),
+        'emotion': random.choice(EMOTIONS),
+        'stable_series': is_stable,
+        'series': series,
+        'email': email,
+        'email_from': email_from,
+        'email_to': email_to,
+        'email_reply_to': email_reply_to,
+        'email_tags': email_tags,
+        'reno_notes': reno_notes,
+        'first_release': first_release,
+        'publishing_dir_name': publishing_dir_name,
+    })
+    if include_pypi_link:
+        params['pypi_url'] = PYPI_URL_TPL % repo_name
+    else:
+        params['pypi_url'] = None
+
+    response = []
+    if changes_only:
+        response.append(expand_template(CHANGES_ONLY_TPL, params))
+    else:
+        if email:
+            email_header = expand_template(EMAIL_HEADER_TPL.strip(), params)
+            response.append(email_header.lstrip())
+        if is_release_candidate:
+            response.append(expand_template(RELEASE_CANDIDATE_TPL, params))
+        else:
+            header = expand_template(HEADER_RELEASE_TPL.strip(), params)
+            response.append(parawrap.fill(header))
+            response.append(expand_template(CHANGE_RELEASE_TPL, params))
+    return '\n'.join(response)

openstack_releases/rst2txt.py

@@ -0,0 +1,966 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Convert reStructuredText to plain text.
+
+Most of this code is adapted from sphinx.writers.text, Copyright
+2007-2015 by the Sphinx team (see
+git://github.com/sphinx-doc/sphinx). The code is imported here because
+it has to be modified to not rely on other internal Sphinx components
+that we don't have set up, such as a Builder and Application object.
+
+"""
+
+import itertools
+import os
+import re
+import textwrap
+
+from docutils import core
+from docutils import nodes
+from docutils.utils import column_width
+from docutils import writers
+from six.moves import zip_longest
+
+from sphinx import addnodes
+
+
+def convert(source):
+    """Given reStructuredText input, return formatted plain text output."""
+    return core.publish_string(
+        source=source,
+        writer=TextWriter(),
+    )
+
+
+class TextWrapper(textwrap.TextWrapper):
+    """Custom subclass that uses a different word separator regex."""
+
+    wordsep_re = re.compile(
+        r'(\s+|'                                  # any whitespace
+        r'(?<=\s)(?::[a-z-]+:)?`\S+|'             # interpreted text start
+        r'[^\s\w]*\w+[a-zA-Z]-(?=\w+[a-zA-Z])|'   # hyphenated words
+        r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))')   # em-dash
+
+    def _wrap_chunks(self, chunks):
+        """_wrap_chunks(chunks : [string]) -> [string]
+
+        The original _wrap_chunks uses len() to calculate width.
+        This method respects wide/fullwidth characters for width adjustment.
+        """
+        drop_whitespace = getattr(self, 'drop_whitespace', True)  # py25 compat
+        lines = []
+        if self.width <= 0:
+            raise ValueError("invalid width %r (must be > 0)" % self.width)
+
+        chunks.reverse()
+
+        while chunks:
+            cur_line = []
+            cur_len = 0
+
+            if lines:
+                indent = self.subsequent_indent
+            else:
+                indent = self.initial_indent
+
+            width = self.width - column_width(indent)
+
+            if drop_whitespace and chunks[-1].strip() == '' and lines:
+                del chunks[-1]
+
+            while chunks:
+                l = column_width(chunks[-1])
+
+                if cur_len + l <= width:
+                    cur_line.append(chunks.pop())
+                    cur_len += l
+
+                else:
+                    break
+
+            if chunks and column_width(chunks[-1]) > width:
+                self._handle_long_word(chunks, cur_line, cur_len, width)
+
+            if drop_whitespace and cur_line and cur_line[-1].strip() == '':
+                del cur_line[-1]
+
+            if cur_line:
+                lines.append(indent + ''.join(cur_line))
+
+        return lines
+
+    def _break_word(self, word, space_left):
+        """_break_word(word : string, space_left : int) -> (string, string)
+
+        Break line by unicode width instead of len(word).
+        """
+        total = 0
+        for i, c in enumerate(word):
+            total += column_width(c)
+            if total > space_left:
+                return word[:i - 1], word[i - 1:]
+        return word, ''
+
+    def _split(self, text):
+        """_split(text : string) -> [string]
+
+        Override original method that only split by 'wordsep_re'.
+        This '_split' split wide-characters into chunk by one character.
+        """
+        def split(t):
+            return textwrap.TextWrapper._split(self, t)
+        chunks = []
+        for chunk in split(text):
+            for w, g in itertools.groupby(chunk, column_width):
+                if w == 1:
+                    chunks.extend(split(''.join(g)))
+                else:
+                    chunks.extend(list(g))
+        return chunks
+
+    def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
+        """Deal with long words
+
+        _handle_long_word(chunks : [string],
+                          cur_line : [string],
+                          cur_len : int, width : int)
+
+        Override original method for using self._break_word() instead
+        of slice.
+
+        """
+        space_left = max(width - cur_len, 1)
+        if self.break_long_words:
+            l, r = self._break_word(reversed_chunks[-1], space_left)
+            cur_line.append(l)
+            reversed_chunks[-1] = r
+
+        elif not cur_line:
+            cur_line.append(reversed_chunks.pop())
+
+
+MAXWIDTH = 70
+STDINDENT = 3
+
+
+def my_wrap(text, width=MAXWIDTH, **kwargs):
+    w = TextWrapper(width=width, **kwargs)
+    return w.wrap(text)
+
+
+class TextWriter(writers.Writer):
+    supported = ('text',)
+    settings_spec = ('No options here.', '', ())
+    settings_defaults = {}
+
+    output = None
+
+    def __init__(self):
+        writers.Writer.__init__(self)
+        self.translator_class = TextTranslator
+
+    def translate(self):
+        visitor = self.translator_class(self.document)
+        self.document.walkabout(visitor)
+        self.output = visitor.body
+
+
+class TextTranslator(nodes.NodeVisitor):
+    sectionchars = '*=-~"+`'
+
+    def __init__(self, document):
+        nodes.NodeVisitor.__init__(self, document)
+
+        self.nl = os.linesep
+        self.states = [[]]
+        self.stateindent = [0]
+        self.list_counter = []
+        self.sectionlevel = 0
+        self.lineblocklevel = 0
+        self.table = None
+
+    def add_text(self, text):
+        self.states[-1].append((-1, text))
+
+    def new_state(self, indent=STDINDENT):
+        self.states.append([])
+        self.stateindent.append(indent)
+
+    def end_state(self, wrap=True, end=[''], first=None):
+        content = self.states.pop()
+        maxindent = sum(self.stateindent)
+        indent = self.stateindent.pop()
+        result = []
+        toformat = []
+
+        def do_format():
+            if not toformat:
+                return
+            if wrap:
+                res = my_wrap(''.join(toformat), width=MAXWIDTH - maxindent)
+            else:
+                res = ''.join(toformat).splitlines()
+            if end:
+                res += end
+            result.append((indent, res))
+        for itemindent, item in content:
+            if itemindent == -1:
+                toformat.append(item)
+            else:
+                do_format()
+                result.append((indent + itemindent, item))
+                toformat = []
+        do_format()
+        if first is not None and result:
+            itemindent, item = result[0]
+            result_rest, result = result[1:], []
+            if item:
+                toformat = [first + ' '.join(item)]
+                do_format()  # re-create `result` from `toformat`
+                _dummy, new_item = result[0]
+                result.insert(0, (itemindent - indent, [new_item[0]]))
+                result[1] = (itemindent, new_item[1:])
+                result.extend(result_rest)
+        self.states[-1].extend(result)
+
+    def visit_document(self, node):
+        self.new_state(0)
+
+    def depart_document(self, node):
+        self.end_state()
+        self.body = self.nl.join(line and (' ' * indent + line)
+                                 for indent, lines in self.states[0]
+                                 for line in lines)
+        # XXX header/footer?
+
+    def visit_highlightlang(self, node):
+        raise nodes.SkipNode
+
+    def visit_section(self, node):
+        self._title_char = self.sectionchars[self.sectionlevel]
+        self.sectionlevel += 1
+
+    def depart_section(self, node):
+        self.sectionlevel -= 1
+
+    def visit_topic(self, node):
+        self.new_state(0)
+
+    def depart_topic(self, node):
+        self.end_state()
+
+    visit_sidebar = visit_topic
+    depart_sidebar = depart_topic
+
+    def visit_rubric(self, node):
+        self.new_state(0)
+        self.add_text('-[ ')
+
+    def depart_rubric(self, node):
+        self.add_text(' ]-')
+        self.end_state()
+
+    def visit_compound(self, node):
+        pass
+
+    def depart_compound(self, node):
+        pass
+
+    def visit_glossary(self, node):
+        pass
+
+    def depart_glossary(self, node):
+        pass
+
+    def visit_title(self, node):
+        if isinstance(node.parent, nodes.Admonition):
+            self.add_text(node.astext() + ': ')
+            raise nodes.SkipNode
+        self.new_state(0)
+
+    def depart_title(self, node):
+        if isinstance(node.parent, nodes.section):
+            char = self._title_char
+        else:
+            char = '^'
+        text = ''.join(x[1] for x in self.states.pop() if x[0] == -1)
+        self.stateindent.pop()
+        self.states[-1].append(
+            (0, ['', text, '%s' % (char * column_width(text)), '']))
+
+    def visit_subtitle(self, node):
+        pass
+
+    def depart_subtitle(self, node):
+        pass
+
+    def visit_attribution(self, node):
+        self.add_text('-- ')
+
+    def depart_attribution(self, node):
+        pass
+
+    def visit_desc(self, node):
+        pass
+
+    def depart_desc(self, node):
+        pass
+
+    def visit_desc_signature(self, node):
+        self.new_state(0)
+        if node.parent['objtype'] in ('class', 'exception'):
+            self.add_text('%s ' % node.parent['objtype'])
+
+    def depart_desc_signature(self, node):
+        # XXX: wrap signatures in a way that makes sense
+        self.end_state(wrap=False, end=None)
+
+    def visit_desc_name(self, node):
+        pass
+
+    def depart_desc_name(self, node):
+        pass
+
+    def visit_desc_addname(self, node):
+        pass
+
+    def depart_desc_addname(self, node):
+        pass
+
+    def visit_desc_type(self, node):
+        pass
+
+    def depart_desc_type(self, node):
+        pass
+
+    def visit_desc_returns(self, node):
+        self.add_text(' -> ')
+
+    def depart_desc_returns(self, node):
+        pass
+
+    def visit_desc_parameterlist(self, node):
+        self.add_text('(')
+        self.first_param = 1
+
+    def depart_desc_parameterlist(self, node):
+        self.add_text(')')
+
+    def visit_desc_parameter(self, node):
+        if not self.first_param:
+            self.add_text(', ')
+        else:
+            self.first_param = 0
+        self.add_text(node.astext())
+        raise nodes.SkipNode
+
+    def visit_desc_optional(self, node):
+        self.add_text('[')
+
+    def depart_desc_optional(self, node):
+        self.add_text(']')
+
+    def visit_desc_annotation(self, node):
+        pass
+
+    def depart_desc_annotation(self, node):
+        pass
+
+    def visit_desc_content(self, node):
+        self.new_state()
+        self.add_text(self.nl)
+
+    def depart_desc_content(self, node):
+        self.end_state()
+
+    def visit_figure(self, node):
+        self.new_state()
+
+    def depart_figure(self, node):
+        self.end_state()
+
+    def visit_caption(self, node):
+        pass
+
+    def depart_caption(self, node):
+        pass
+
+    def visit_productionlist(self, node):
+        self.new_state()
+        names = []
+        for production in node:
+            names.append(production['tokenname'])
+        maxlen = max(len(name) for name in names)
+        lastname = None
+        for production in node:
+            if production['tokenname']:
+                self.add_text(production['tokenname'].ljust(maxlen) + ' ::=')
+                lastname = production['tokenname']
+            elif lastname is not None:
+                self.add_text('%s    ' % (' ' * len(lastname)))
+            self.add_text(production.astext() + self.nl)
+        self.end_state(wrap=False)
+        raise nodes.SkipNode
+
+    def visit_footnote(self, node):
+        self._footnote = node.children[0].astext().strip()
+        self.new_state(len(self._footnote) + 3)
+
+    def depart_footnote(self, node):
+        self.end_state(first='[%s] ' % self._footnote)
+
+    def visit_citation(self, node):
+        if len(node) and isinstance(node[0], nodes.label):
+            self._citlabel = node[0].astext()
+        else:
+            self._citlabel = ''
+        self.new_state(len(self._citlabel) + 3)
+
+    def depart_citation(self, node):
+        self.end_state(first='[%s] ' % self._citlabel)
+
+    def visit_label(self, node):
+        raise nodes.SkipNode
+
+    def visit_legend(self, node):
+        pass
+
+    def depart_legend(self, node):
+        pass
+
+    # XXX: option list could use some better styling
+
+    def visit_option_list(self, node):
+        pass
+
+    def depart_option_list(self, node):
+        pass
+
+    def visit_option_list_item(self, node):
+        self.new_state(0)
+
+    def depart_option_list_item(self, node):
+        self.end_state()
+
+    def visit_option_group(self, node):
+        self._firstoption = True
+
+    def depart_option_group(self, node):
+        self.add_text('     ')
+
+    def visit_option(self, node):
+        if self._firstoption:
+            self._firstoption = False
+        else:
+            self.add_text(', ')
+
+    def depart_option(self, node):
+        pass
+
+    def visit_option_string(self, node):
+        pass
+
+    def depart_option_string(self, node):
+        pass
+
+    def visit_option_argument(self, node):
+        self.add_text(node['delimiter'])
+
+    def depart_option_argument(self, node):
+        pass
+
+    def visit_description(self, node):
+        pass
+
+    def depart_description(self, node):
+        pass
+
+    def visit_tabular_col_spec(self, node):
+        raise nodes.SkipNode
+
+    def visit_colspec(self, node):
+        self.table[0].append(node['colwidth'])
+        raise nodes.SkipNode
+
+    def visit_tgroup(self, node):
+        pass
+
+    def depart_tgroup(self, node):
+        pass
+
+    def visit_thead(self, node):
+        pass
+
+    def depart_thead(self, node):
+        pass
+
+    def visit_tbody(self, node):
+        self.table.append('sep')
+
+    def depart_tbody(self, node):
+        pass
+
+    def visit_row(self, node):
+        self.table.append([])
+
+    def depart_row(self, node):
+        pass
+
+    def visit_entry(self, node):
+        if 'morerows' in node or 'morecols' in node:
+            raise NotImplementedError('Column or row spanning cells are '
+                                      'not implemented.')
+        self.new_state(0)
+
+    def depart_entry(self, node):
+        text = self.nl.join(self.nl.join(x[1]) for x in self.states.pop())
+        self.stateindent.pop()
+        self.table[-1].append(text)
+
+    def visit_table(self, node):
+        if self.table:
+            raise NotImplementedError('Nested tables are not supported.')
+        self.new_state(0)
+        self.table = [[]]
+
+    def depart_table(self, node):
+        lines = self.table[1:]
+        fmted_rows = []
+        colwidths = self.table[0]
+        realwidths = colwidths[:]
+        separator = 0
+        # don't allow paragraphs in table cells for now
+        for line in lines:
+            if line == 'sep':
+                separator = len(fmted_rows)
+            else:
+                cells = []
+                for i, cell in enumerate(line):
+                    par = my_wrap(cell, width=colwidths[i])
+                    if par:
+                        maxwidth = max(column_width(x) for x in par)
+                    else:
+                        maxwidth = 0
+                    realwidths[i] = max(realwidths[i], maxwidth)
+                    cells.append(par)
+                fmted_rows.append(cells)
+
+        def writesep(char='-'):
+            out = ['+']
+            for width in realwidths:
+                out.append(char * (width + 2))
+                out.append('+')
+            self.add_text(''.join(out) + self.nl)
+
+        def writerow(row):
+            lines = zip_longest(*row)
+            for line in lines:
+                out = ['|']
+                for i, cell in enumerate(line):
+                    if cell:
+                        adjust_len = len(cell) - column_width(cell)
+                        out.append(' ' + cell.ljust(
+                            realwidths[i] + 1 + adjust_len))
+                    else:
+                        out.append(' ' * (realwidths[i] + 2))
+                    out.append('|')
+                self.add_text(''.join(out) + self.nl)
+
+        for i, row in enumerate(fmted_rows):
+            if separator and i == separator:
+                writesep('=')
+            else:
+                writesep('-')
+            writerow(row)
+        writesep('-')
+        self.table = None
+        self.end_state(wrap=False)
+
+    def visit_acks(self, node):
+        self.new_state(0)
+        self.add_text(
+            ', '.join(n.astext() for n in node.children[0].children) +
+            '.')
+        self.end_state()
+        raise nodes.SkipNode
+
+    def visit_image(self, node):
+        if 'alt' in node.attributes:
+            self.add_text('[image: %s]' % node['alt'])
+        self.add_text('[image]')
+        raise nodes.SkipNode
+
+    def visit_transition(self, node):
+        indent = sum(self.stateindent)
+        self.new_state(0)
+        self.add_text('=' * (MAXWIDTH - indent))
+        self.end_state()
+        raise nodes.SkipNode
+
+    def visit_bullet_list(self, node):
+        self.list_counter.append(-1)
+
+    def depart_bullet_list(self, node):
+        self.list_counter.pop()
+
+    def visit_enumerated_list(self, node):
+        self.list_counter.append(node.get('start', 1) - 1)
+
+    def depart_enumerated_list(self, node):
+        self.list_counter.pop()
+
+    def visit_definition_list(self, node):
+        self.list_counter.append(-2)
+
+    def depart_definition_list(self, node):
+        self.list_counter.pop()
+
+    def visit_list_item(self, node):
+        if self.list_counter[-1] == -1:
+            # bullet list
+            self.new_state(2)
+        elif self.list_counter[-1] == -2:
+            # definition list
+            pass
+        else:
+            # enumerated list
+            self.list_counter[-1] += 1
+            self.new_state(len(str(self.list_counter[-1])) + 2)
+
+    def depart_list_item(self, node):
+        if self.list_counter[-1] == -1:
+            self.end_state(first='* ')
+        elif self.list_counter[-1] == -2:
+            pass
+        else:
+            self.end_state(first='%s. ' % self.list_counter[-1])
+
+    def visit_definition_list_item(self, node):
+        self._li_has_classifier = len(node) >= 2 and \
+            isinstance(node[1], nodes.classifier)
+
+    def depart_definition_list_item(self, node):
+        pass
+
+    def visit_term(self, node):
+        self.new_state(0)
+
+    def depart_term(self, node):
+        if not self._li_has_classifier:
+            self.end_state(end=None)
+
+    def visit_termsep(self, node):
+        self.add_text(', ')
+        raise nodes.SkipNode
+
+    def visit_classifier(self, node):
+        self.add_text(' : ')
+
+    def depart_classifier(self, node):
+        self.end_state(end=None)
+
+    def visit_definition(self, node):
+        self.new_state()
+
+    def depart_definition(self, node):
+        self.end_state()
+
+    def visit_field_list(self, node):
+        pass
+
+    def depart_field_list(self, node):
+        pass
+
+    def visit_field(self, node):
+        pass
+
+    def depart_field(self, node):
+        pass
+
+    def visit_field_name(self, node):
+        self.new_state(0)
+
+    def depart_field_name(self, node):
+        self.add_text(':')
+        self.end_state(end=None)
+
+    def visit_field_body(self, node):
+        self.new_state()
+
+    def depart_field_body(self, node):
+        self.end_state()
+
+    def visit_centered(self, node):
+        pass
+
+    def depart_centered(self, node):
+        pass
+
+    def visit_hlist(self, node):
+        pass
+
+    def depart_hlist(self, node):
+        pass
+
+    def visit_hlistcol(self, node):
+        pass
+
+    def depart_hlistcol(self, node):
+        pass
+
+    def visit_admonition(self, node):
+        self.new_state(0)
+
+    def depart_admonition(self, node):
+        self.end_state()
+
+    def _visit_admonition(self, node):
+        self.new_state(2)
+
+    def _make_depart_admonition(name):
+        def depart_admonition(self, node):
+            self.end_state(first=name.title() + ': ')
+        return depart_admonition
+
+    visit_attention = _visit_admonition
+    depart_attention = _make_depart_admonition('attention')
+    visit_caution = _visit_admonition
+    depart_caution = _make_depart_admonition('caution')
+    visit_danger = _visit_admonition
+    depart_danger = _make_depart_admonition('danger')
+    visit_error = _visit_admonition
+    depart_error = _make_depart_admonition('error')
+    visit_hint = _visit_admonition
+    depart_hint = _make_depart_admonition('hint')
+    visit_important = _visit_admonition
+    depart_important = _make_depart_admonition('important')
+    visit_note = _visit_admonition
+    depart_note = _make_depart_admonition('note')
+    visit_tip = _visit_admonition
+    depart_tip = _make_depart_admonition('tip')
+    visit_warning = _visit_admonition
+    depart_warning = _make_depart_admonition('warning')
+    visit_seealso = _visit_admonition
+    depart_seealso = _make_depart_admonition('seealso')
+
+    def visit_versionmodified(self, node):
+        self.new_state(0)
+
+    def depart_versionmodified(self, node):
+        self.end_state()
+
+    def visit_literal_block(self, node):
+        self.new_state()
+
+    def depart_literal_block(self, node):
+        self.end_state(wrap=False)
+
+    def visit_doctest_block(self, node):
+        self.new_state(0)
+
+    def depart_doctest_block(self, node):
+        self.end_state(wrap=False)
+
+    def visit_line_block(self, node):
+        self.new_state()
+        self.lineblocklevel += 1
+
+    def depart_line_block(self, node):
+        self.lineblocklevel -= 1
+        self.end_state(wrap=False, end=None)
+        if not self.lineblocklevel:
+            self.add_text('\n')
+
+    def visit_line(self, node):
+        pass
+
+    def depart_line(self, node):
+        self.add_text('\n')
+
+    def visit_block_quote(self, node):
+        self.new_state()
+
+    def depart_block_quote(self, node):
+        self.end_state()
+
+    def visit_compact_paragraph(self, node):
+        pass
+
+    def depart_compact_paragraph(self, node):
+        pass
+
+    def visit_paragraph(self, node):
+        if not isinstance(node.parent, nodes.Admonition) or \
+           isinstance(node.parent, addnodes.seealso):
+            self.new_state(0)
+
+    def depart_paragraph(self, node):
+        if not isinstance(node.parent, nodes.Admonition) or \
+           isinstance(node.parent, addnodes.seealso):
+            self.end_state()
+
+    def visit_target(self, node):
+        pass
+
+    def depart_target(self, node):
+        self.add_text(' (%s)' % node['refuri'])
+
+    def visit_index(self, node):
+        raise nodes.SkipNode
+
+    def visit_toctree(self, node):
+        raise nodes.SkipNode
+
+    def visit_substitution_definition(self, node):
+        raise nodes.SkipNode
+
+    def visit_pending_xref(self, node):
+        pass
+
+    def depart_pending_xref(self, node):
+        pass
+
+    def visit_reference(self, node):
+        pass
+
+    def depart_reference(self, node):
+        pass
+
+    def visit_number_reference(self, node):
+        text = nodes.Text(node.get('title', '#'))
+        self.visit_Text(text)
+        raise nodes.SkipNode
+
+    def visit_download_reference(self, node):
+        pass
+
+    def depart_download_reference(self, node):
+        pass
+
+    def visit_emphasis(self, node):
+        self.add_text('*')
+
+    def depart_emphasis(self, node):
+        self.add_text('*')
+
+    def visit_literal_emphasis(self, node):
+        self.add_text('*')
+
+    def depart_literal_emphasis(self, node):
+        self.add_text('*')
+
+    def visit_strong(self, node):
+        self.add_text('**')
+
+    def depart_strong(self, node):
+        self.add_text('**')
+
+    def visit_literal_strong(self, node):
+        self.add_text('**')
+
+    def depart_literal_strong(self, node):
+        self.add_text('**')
+
+    def visit_abbreviation(self, node):
+        self.add_text('')
+
+    def depart_abbreviation(self, node):
+        if node.hasattr('explanation'):
+            self.add_text(' (%s)' % node['explanation'])
+
+    def visit_title_reference(self, node):
+        self.add_text('*')
+
+    def depart_title_reference(self, node):
+        self.add_text('*')
+
+    def visit_literal(self, node):
+        self.add_text('"')
+
+    def depart_literal(self, node):
+        self.add_text('"')
+
+    def visit_subscript(self, node):
+        self.add_text('_')
+
+    def depart_subscript(self, node):
+        pass
+
+    def visit_superscript(self, node):
+        self.add_text('^')
+
+    def depart_superscript(self, node):
+        pass
+
+    def visit_footnote_reference(self, node):
+        self.add_text('[%s]' % node.astext())
+        raise nodes.SkipNode
+
+    def visit_citation_reference(self, node):
+        self.add_text('[%s]' % node.astext())
+        raise nodes.SkipNode
+
+    def visit_Text(self, node):
+        self.add_text(node.astext())
+
+    def depart_Text(self, node):
+        pass
+
+    def visit_generated(self, node):
+        pass
+
+    def depart_generated(self, node):
+        pass
+
+    def visit_inline(self, node):
+        if 'xref' in node['classes'] or 'term' in node['classes']:
+            self.add_text('*')
+
+    def depart_inline(self, node):
+        if 'xref' in node['classes'] or 'term' in node['classes']:
+            self.add_text('*')
+
+    def visit_container(self, node):
+        pass
+
+    def depart_container(self, node):
+        pass
+
+    def visit_problematic(self, node):
+        pass
+
+    def depart_problematic(self, node):
+        pass
+
+    def visit_system_message(self, node):
+        raise nodes.SkipNode
+
+    def visit_comment(self, node):
+        raise nodes.SkipNode
+
+    def visit_meta(self, node):
+        # only valid for HTML
+        raise nodes.SkipNode
+
+    def visit_raw(self, node):
+        if 'text' in node.get('format', '').split():
+            self.body.append(node.astext())
+        raise nodes.SkipNode
+
+    def visit_math(self, node):
+        raise nodes.SkipNode
+
+    visit_math_block = visit_math
+
+    def unknown_visit(self, node):
+        raise NotImplementedError('Unknown node: ' + node.__class__.__name__)

requirements.txt

@@ -13,3 +13,9 @@ tqdm
 packaging>=15.2
 mwclient==0.8.1
 jsonschema>=2.6.0
+
+# For release notes generation.
+Jinja2>=2.6  # BSD License (3 clause)
+parawrap
+reno>=2.0.0
+sphinx>=1.6.2  # BSD

setup.cfg

@@ -39,6 +39,7 @@ console_scripts =
     propose-library-branches = openstack_releases.cmds.propose_library_branches:main
     edit-deliverable = openstack_releases.cmds.edit_deliverable:main
     send-mail = openstack_releases.cmds.mail:main
+    release-notes = openstack_releases.cmds.release_notes:main
 
 [extras]
 sphinxext =