Convert docstrings to md

python docstrings are assumed to be RST, while for OpenAPI we need MD.
Convert controller description from rst->html->md using docutils.

Change-Id: I04acab45f5cf9d05b5d7158e484a14c4343bdb24

.pre-commit-config.yaml

@@ -37,4 +37,5 @@ repos:
         pass_filenames: false
         additional_dependencies:
           - types-decorator
+          - types-docutils
           - types-PyYAML

codegenerator/common/rust.py

@@ -23,6 +23,9 @@ from codegenerator import model
 from codegenerator import common
 
 
+CODEBLOCK_RE = re.compile(r"```(\w*)$")
+
+
 class Boolean(BasePrimitiveType):
     """Basic Boolean"""
 
@@ -1089,12 +1092,14 @@ def sanitize_rust_docstrings(doc: str | None) -> str | None:
     code_block_open: bool = False
     lines: list[str] = []
     for line in doc.split("\n"):
-        if line.endswith("```"):
+        m = re.search(CODEBLOCK_RE, line)
+        if m and m.groups():
             if not code_block_open:
+                code_block_open = True
                 # Rustdoc defaults to rust code for code blocks. To prevent
                 # this explicitly add `text`
-                code_block_open = True
+                if m.group(1) == "":
-                line = line + "text"
+                    line = line + "text"
             else:
                 code_block_open = False
         lines.append(line)

codegenerator/openapi/base.py

@@ -24,6 +24,7 @@ from codegenerator.common.schema import ParameterSchema
 from codegenerator.common.schema import PathSchema
 from codegenerator.common.schema import SpecSchema
 from codegenerator.common.schema import TypeSchema
+from codegenerator.openapi.utils import rst_to_md
 from openapi_core import Spec
 from ruamel.yaml.scalarstring import LiteralScalarString
 from ruamel.yaml import YAML
@@ -547,6 +548,7 @@ class OpenStackServerSourceBase:
         if mode != "action":
             doc = inspect.getdoc(func)
             if doc and not operation_spec.description:
+                doc = rst_to_md(doc)
                 operation_spec.description = LiteralScalarString(doc)
         if operation_spec.description:
             # Reading spec from yaml file it was converted back to regular

codegenerator/openapi/utils.py

@@ -14,6 +14,7 @@ import logging
 import re
 
 from bs4 import BeautifulSoup
+from docutils.core import publish_parts
 from codegenerator.common.schema import TypeSchema
 from markdownify import markdownify as md
 from ruamel.yaml.scalarstring import LiteralScalarString
@@ -540,5 +541,18 @@ def _get_schema_candidates(
     return (schema_specs, action_name)
 
 
+def rst_to_md(content: str) -> str:
+    """Convert RST string to Markdown"""
+    # Sadly the only reasonably usable library to do direct conversion is now
+    # also abandoned (rst-to-myst). The only alternative known at the moment of
+    # writing is to use docutils/sphinx directly and convert rst to html to
+    # convert it then to md. Since I do not know how to involve sphinx from the
+    # python code the only remaining way seem to be docutils (but with it certain
+    # sphinx specifics are lost)
+    html = publish_parts(content, writer_name="html")["html_body"]
+    res = get_sanitized_description(html)
+    return res
+
+
 def get_sanitized_description(descr: str) -> LiteralScalarString:
     return LiteralScalarString(md(descr, escape_underscores=False).rstrip())

requirements.txt

@@ -11,6 +11,8 @@ markdownify>=0.11 # MIT
 mdformat>=0.7 # MIT
 # Get sphinx docstrings from source code
 sphinx>=2.0.0,!=2.1.0 # BSD
+# Convert rst docstrings to html (openapi)
+docutils>=0.20 # MIT
 wsgi-intercept>=1.13 # MIT
 wsme>=0.12 # MIT
 oslotest>=5.0.0 # Apache-2.0