Allow comments in variable files to be rendered in docs

This change will allow all comments in our variable files to be rendered
normally within our documentation. This will allow folks reading our
documentation to benefit from information we may put in the various
files.

This patch adds ruamel.yaml as new dependency for building documentation
and we won't need to include a README.md file into the documentation
role index anymore.

Depends-On: https://review.opendev.org/c/openstack/requirements/+/795270
Upstream-rpm-master: https://review.rdoproject.org/r/c/openstack/tripleo-validations-distgit/+/34031

Co-Authored-By: Kevin Carter <kecarter@redhat.com>

Signed-off-by: Gael Chamoulaud (Strider) <gchamoul@redhat.com>
Change-Id: I88ac702a9eedc53fdc88a2dbb5422b94322b68b4
This commit is contained in:
Gael Chamoulaud (Strider) 2021-06-08 11:12:51 +02:00
parent 86bdd14bc1
commit 933d98402f
No known key found for this signature in database
GPG Key ID: 4119D0305C651D66
2 changed files with 123 additions and 73 deletions

View File

@ -4,4 +4,5 @@ openstackdocstheme>=2.2.2 # Apache-2.0
reno>=3.1.0 # Apache-2.0 reno>=3.1.0 # Apache-2.0
doc8>=0.8.0 # Apache-2.0 doc8>=0.8.0 # Apache-2.0
bashate>=0.6.0 # Apache-2.0 bashate>=0.6.0 # Apache-2.0
ruamel.yaml>=0.15.5 # MIT
six>=1.11.0 # MIT six>=1.11.0 # MIT

View File

@ -14,7 +14,7 @@
# under the License. # under the License.
import importlib.util import imp
import os import os
from docutils import core from docutils import core
@ -23,7 +23,48 @@ from docutils.parsers.rst import Directive
from docutils.parsers import rst from docutils.parsers import rst
from docutils.writers.html4css1 import Writer from docutils.writers.html4css1 import Writer
from sphinx import addnodes
import yaml import yaml
from ruamel.yaml import YAML as RYAML
try:
import io
StringIO = io.StringIO
except ImportError:
import StringIO
class DocYaml(RYAML):
def _license_filter(self, data):
"""This will filter out our boilerplate license heading in return data.
The filter is used to allow documentation we're creating in variable
files to be rendered more beautifully.
"""
lines = list()
mark = True
for line in data.splitlines():
if '# Copyright' in line:
mark = False
if mark:
lines.append(line)
if '# under the License' in line:
mark = True
return '\n'.join(lines)
def dump(self, data, stream=None, **kw):
if not stream:
stream = StringIO()
try:
RYAML.dump(self, data, stream, **kw)
return self._license_filter(stream.getvalue().strip())
finally:
stream.close()
DOCYAML = DocYaml()
DOCYAML.default_flow_style = False
class AnsibleAutoPluginDirective(Directive): class AnsibleAutoPluginDirective(Directive):
@ -93,16 +134,11 @@ class AnsibleAutoPluginDirective(Directive):
@staticmethod @staticmethod
def load_module(filename): def load_module(filename):
module_spec = importlib.util.spec_from_file_location( return imp.load_source('__ansible_module__', filename)
'__ansible_module__', filename
)
module = importlib.util.module_from_spec(module_spec)
module_spec.loader.exec_module(module)
return module
@staticmethod @staticmethod
def build_documentation(module): def build_documentation(module):
docs = yaml.safe_load(module.DOCUMENTATION) docs = DOCYAML.load(module.DOCUMENTATION)
doc_data = dict() doc_data = dict()
doc_data['docs'] = docs['description'] doc_data['docs'] = docs['description']
doc_data['author'] = docs.get('author', list()) doc_data['author'] = docs.get('author', list())
@ -111,12 +147,10 @@ class AnsibleAutoPluginDirective(Directive):
@staticmethod @staticmethod
def build_examples(module): def build_examples(module):
examples = yaml.safe_load(module.EXAMPLES) examples = DOCYAML.load(module.EXAMPLES)
return_examples = list() return_examples = list()
for example in examples: for example in examples:
return_examples.append( return_examples.append(DOCYAML.dump([example]))
yaml.safe_dump([example], default_flow_style=False)
)
return return_examples return return_examples
def _raw_html_block(self, data): def _raw_html_block(self, data):
@ -140,10 +174,7 @@ class AnsibleAutoPluginDirective(Directive):
def _literal_block(data, language='yaml', dump_data=True): def _literal_block(data, language='yaml', dump_data=True):
if dump_data: if dump_data:
literal = nodes.literal_block( literal = nodes.literal_block(
text=yaml.safe_dump( text=DOCYAML.dump(data)
data,
default_flow_style=False
)
) )
else: else:
literal = nodes.literal_block(text=data) literal = nodes.literal_block(text=data)
@ -174,92 +205,110 @@ class AnsibleAutoPluginDirective(Directive):
def _run_role(self, role): def _run_role(self, role):
section = self._section_block( section = self._section_block(
title='Role Documentation', title="Role Documentation",
text='Welcome to the "{}" role documentation.'.format( text='Welcome to the "{}" role documentation.'.format(
os.path.basename(role) os.path.basename(role)
) ),
) )
molecule_defaults = None
abspath_role = os.path.dirname(os.path.abspath(role)) abspath_role = os.path.dirname(os.path.abspath(role))
molecule_shared_file = os.path.join(os.path.dirname(abspath_role), molecule_shared_file = os.path.join(
".config/molecule/config.yml") os.path.dirname(abspath_role), ".config/molecule/config.yml"
)
if os.path.exists(molecule_shared_file): if os.path.exists(molecule_shared_file):
with open(molecule_shared_file) as msf: with open(molecule_shared_file) as msf:
molecule_defaults = yaml.safe_load(msf.read()) molecule_defaults = DOCYAML.load(msf.read())
defaults_file = os.path.join(role, 'defaults', 'main.yml') defaults_file = os.path.join(role, "defaults", "main.yml")
if os.path.exists(defaults_file): if os.path.exists(defaults_file):
with open(defaults_file) as f: with open(defaults_file) as f:
role_defaults = yaml.safe_load(f.read()) role_defaults = DOCYAML.load(f.read())
section.append( section.append(
self._yaml_section( self._yaml_section(
to_yaml_data=role_defaults, to_yaml_data=role_defaults,
section_title='Role Defaults', section_title="Role Defaults",
section_text='This section highlights all of the defaults' section_text="This section highlights all of the defaults"
' and variables set within the "{}"' ' and variables set within the "{}"'
' role.'.format( " role.".format(os.path.basename(role)),
os.path.basename(role)
)
) )
) )
vars_path = os.path.join(role, 'vars') vars_path = os.path.join(role, "vars")
if os.path.exists(vars_path): if os.path.exists(vars_path):
for v_file in os.listdir(vars_path): for v_file in os.listdir(vars_path):
vars_file = os.path.join(vars_path, v_file) vars_file = os.path.join(vars_path, v_file)
with open(vars_file) as f: with open(vars_file) as f:
vars_values = yaml.safe_load(f.read()) vars_values = DOCYAML.load(f.read())
section.append( section.append(
self._yaml_section( self._yaml_section(
to_yaml_data=vars_values, to_yaml_data=vars_values,
section_title='Role Variables: {}'.format(v_file) section_title="Role Variables: {}".format(v_file),
) )
) )
test_list = nodes.field_list()
test_section = self._section_block( test_section = self._section_block(
title='Molecule Scenarios', title="Molecule Scenarios",
text='Molecule is being used to test the "{}" role. The' text='Molecule is being used to test the "{}" role. The'
' following section highlights the drivers in service' " following section highlights the drivers in service"
' and provides an example playbook showing how the role' " and provides an example playbook showing how the role"
' is leveraged.'.format( " is leveraged.".format(os.path.basename(role)),
os.path.basename(role)
)
) )
molecule_path = os.path.join(role, 'molecule')
molecule_path = os.path.join(role, "molecule")
if os.path.exists(molecule_path): if os.path.exists(molecule_path):
for test in os.listdir(molecule_path): for test in os.listdir(molecule_path):
molecule_section = self._section_block( molecule_section = self._section_block(
title='Scenario: {}'.format(test) title="Scenario: {}".format(test)
) )
molecule_file = os.path.join( molecule_file = os.path.join(molecule_path, test, "molecule.yml")
molecule_path, if not os.path.exists(molecule_file):
test, continue
'molecule.yml'
)
with open(molecule_file) as f:
molecule_conf = yaml.safe_load(f.read())
with open(molecule_file) as f:
molecule_conf = DOCYAML.load(f.read())
# if molecule.yml file from the scenarios, we get the
# information from the molecule shared configuration file.
if not molecule_conf: if not molecule_conf:
molecule_conf = molecule_defaults molecule_conf = molecule_defaults
molecule_section.append(
self._yaml_section(
to_yaml_data=molecule_conf,
section_title='Example {} configuration'.format(test)
)
)
default_playbook = [molecule_path, test, 'converge.yml']
provisioner_data = None
# Now that we use a shared molecule configuration file, the # Now that we use a shared molecule configuration file, the
# molecule.yml file in the role scenarios could be empty or # molecule.yml file in the role scenarios could be empty or
# contains only overriding keys. # contains only overriding keys.
if molecule_conf: driver_data = molecule_conf.get('driver',
provisioner_data = molecule_conf.get('provisioner') molecule_defaults.get('driver'))
else:
provisioner_data = molecule_defaults.get('provisioner') if driver_data:
molecule_section.append(
nodes.field_name(text="Driver: {}".format(driver_data["name"]))
)
options = driver_data.get("options")
if options:
molecule_section.append(
self._yaml_section(
to_yaml_data=options, section_title="Molecule Options"
)
)
platforms_data = molecule_conf.get('platforms',
molecule_defaults.get('platforms'))
if platforms_data:
molecule_section.append(
self._yaml_section(
to_yaml_data=platforms_data,
section_title="Molecule Platform(s)",
)
)
default_playbook = [molecule_path, test, "converge.yml"]
provisioner_data = molecule_conf.get('provisioner',
molecule_defaults.get('provisioner'))
if provisioner_data: if provisioner_data:
inventory = provisioner_data.get('inventory') inventory = provisioner_data.get('inventory')
@ -267,9 +316,10 @@ class AnsibleAutoPluginDirective(Directive):
molecule_section.append( molecule_section.append(
self._yaml_section( self._yaml_section(
to_yaml_data=inventory, to_yaml_data=inventory,
section_title='Molecule Inventory' section_title="Molecule Inventory",
) )
) )
try: try:
converge = provisioner_data['playbooks']['converge'] converge = provisioner_data['playbooks']['converge']
default_playbook = default_playbook[:-1] + [converge] default_playbook = default_playbook[:-1] + [converge]
@ -277,36 +327,35 @@ class AnsibleAutoPluginDirective(Directive):
pass pass
molecule_playbook_path = os.path.join(*default_playbook) molecule_playbook_path = os.path.join(*default_playbook)
with open(molecule_playbook_path) as f: with open(molecule_playbook_path) as f:
molecule_playbook = yaml.safe_load(f.read()) molecule_playbook = DOCYAML.load(f.read())
molecule_section.append( molecule_section.append(
self._yaml_section( self._yaml_section(
to_yaml_data=molecule_playbook, to_yaml_data=molecule_playbook,
section_title='Example {} playbook'.format(test) section_title="Example {} playbook".format(test),
) )
) )
test_section.append(molecule_section) test_list.append(molecule_section)
else: else:
test_section.append(test_list)
section.append(test_section) section.append(test_section)
self.run_returns.append(section) self.run_returns.append(section)
# Document any libraries nested within the role # Document any libraries nested within the role
library_path = os.path.join(role, 'library') library_path = os.path.join(role, "library")
if os.path.exists(library_path): if os.path.exists(library_path):
self.options['documentation'] = True self.options['documentation'] = True
self.options['examples'] = True self.options['examples'] = True
for lib in os.listdir(library_path): for lib in os.listdir(library_path):
if lib.endswith('.py'): if lib.endswith(".py"):
self._run_module( self._run_module(
module=self.load_module( module=self.load_module(
filename=os.path.join( filename=os.path.join(library_path, lib)
library_path,
lib
)
), ),
module_title='Embedded module: {}'.format(lib), module_title="Embedded module: {}".format(lib),
example_title='Examples for embedded module' example_title="Examples for embedded module",
) )
def _run_module(self, module, module_title="Module Documentation", def _run_module(self, module, module_title="Module Documentation",