Convert rst to plaintext for oslo.config output

This lets us use powerful rST roles and directives in our help text
without these being emitted to the configuration file, where they don't
read quite as well. It also fixes our formatting somewhat, which is
nice.

Note that we're doing something funky where we disable line wrapping. We
can't totally disable this due to how rst2txt works, so instead we set
the line length to an arbitrarily long line length, which is taken from
RFC5322 (Internet Message Format) for want of something better. I
personally question whether anyone is configuring this, but that's a
fight for another day.

Also note that this might caught issues for people who are using invalid
rST in their docstrings or use roles/directives in their config options
that aren't part of the standard Sphinx set or oslo.config set. They
will already be seeing the former issues if they are using the
'sphinxext' extension though, and the latter sounds like it would be
fairly rare.

Change-Id: I6c7208f0facfb4f334d7440cb6a9562901543dd3
Signed-off-by: Stephen Finucane <sfinucan@redhat.com>

lower-constraints.txt

@@ -58,6 +58,7 @@ requests-mock==1.5.0
 requestsexceptions==1.2.0
 restructuredtext-lint==1.3.1
 rfc3986==1.2.0
+rst2txt==1.1.0
 six==1.15.0
 smmap==0.9.0
 snowballstemmer==1.2.1

oslo_config/generator.py

@@ -29,7 +29,18 @@ import json
 import logging
 import operator
 import sys
-import textwrap
+
+try:
+    # For oslo.config[rst-generator]
+    from docutils import core as docutils_core
+    from docutils.parsers.rst import nodes as docutils_nodes
+    from docutils.parsers.rst import roles as docutils_roles
+    import rst2txt
+    from sphinx import roles as sphinx_roles
+except ImportError:
+    import textwrap
+
+    rst2txt = None
 
 try:
     # For Python 3.8 and later
@@ -53,6 +64,7 @@ _generator_opts = [
     cfg.IntOpt(
         'wrap-width',
         default=70,
+        min=0,
         help='The maximum length of help lines.'),
     cfg.MultiStrOpt(
         'namespace',
@@ -173,16 +185,43 @@ class _OptFormatter(object):
         :param output_file: a writeable file object
         """
         self.output_file = output_file or sys.stdout
-        self.wrap_width = conf.wrap_width
+        self.wrap_width = conf.wrap_width or 998  # arbitrary line length
         self.minimal = conf.minimal
         self.summarize = conf.summarize
 
+        if rst2txt:
+            # register the default Sphinx roles
+            for rolename, nodeclass in sphinx_roles.generic_docroles.items():
+                generic = docutils_roles.GenericRole(rolename, nodeclass)
+                docutils_roles.register_local_role(rolename, generic)
+
+            for rolename, func in sphinx_roles.specific_docroles.items():
+                docutils_roles.register_local_role(rolename, func)
+
+            # plus our custom roles
+            for rolename in ('oslo.config:option', 'oslo.config:group'):
+                generic = docutils_roles.GenericRole(rolename,
+                                                     docutils_nodes.strong)
+                docutils_roles.register_local_role(rolename, generic)
+
     def _format_help(self, help_text):
         """Format the help for a group or option to the output file.
 
         :param help_text: The text of the help string
         """
-        if self.wrap_width is not None and self.wrap_width > 0:
+        if rst2txt:
+            help_text = docutils_core.publish_string(
+                source=help_text,
+                writer=rst2txt.Writer(),
+                settings_overrides={'wrap_width': self.wrap_width}
+            ).decode()
+
+            lines = ''
+            for line in help_text.splitlines():
+                lines += '# ' + line + '\n' if line else '#\n'
+
+            lines = [lines]
+        elif self.wrap_width > 0:
             wrapped = ""
             for line in help_text.splitlines():
                 text = "\n".join(textwrap.wrap(line, self.wrap_width,
@@ -195,6 +234,7 @@ class _OptFormatter(object):
             lines = [wrapped]
         else:
             lines = ['# ' + help_text + '\n']
+
         return lines
 
     def _get_choice_text(self, choice):

setup.cfg

@@ -22,6 +22,13 @@ classifier =
     Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: Implementation :: CPython
 
+[extras]
+# package dependencies for optional behavior in the config generator.
+# e.g.: oslo.config[generator]
+rst-generator =
+    rst2txt>=1.1.0 # BSD
+    sphinx>=1.8.0,!=2.1.0 # BSD
+
 [files]
 packages =
     oslo_config