Merge "docs: Add documentation for flavor extra specs"

doc/ext/extra_specs.py

@@ -0,0 +1,231 @@
+# Copyright 2020, Red Hat, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Display extra specs in documentation.
+
+Provides a single directive that can be used to list all extra specs validators
+and, thus, document all extra specs that nova recognizes and supports.
+"""
+
+import typing as ty
+
+from docutils import nodes
+from docutils.parsers import rst
+from docutils.parsers.rst import directives
+from docutils import statemachine
+from sphinx import addnodes
+from sphinx import directives as sphinx_directives
+from sphinx import domains
+from sphinx import roles
+from sphinx.util import logging
+from sphinx.util import nodes as sphinx_nodes
+
+from nova.api.validation.extra_specs import base
+from nova.api.validation.extra_specs import validators
+
+LOG = logging.getLogger(__name__)
+
+
+class ExtraSpecXRefRole(roles.XRefRole):
+    """Cross reference a extra spec.
+
+    Example::
+
+        :nova:extra-spec:`hw:cpu_policy`
+    """
+
+    def __init__(self):
+        super(ExtraSpecXRefRole, self).__init__(
+            warn_dangling=True,
+        )
+
+    def process_link(self, env, refnode, has_explicit_title, title, target):
+        # The anchor for the extra spec link is the extra spec name
+        return target, target
+
+
+class ExtraSpecDirective(sphinx_directives.ObjectDescription):
+    """Document an individual extra spec.
+
+    Accepts one required argument - the extra spec name, including the group.
+
+    Example::
+
+        .. extra-spec:: hw:cpu_policy
+    """
+
+    def handle_signature(self, sig, signode):
+        """Transform an option description into RST nodes."""
+        # Insert a node into the output showing the extra spec name
+        signode += addnodes.desc_name(sig, sig)
+        signode['allnames'] = [sig]
+        return sig
+
+    def add_target_and_index(self, firstname, sig, signode):
+        cached_options = self.env.domaindata['nova']['extra_specs']
+        signode['ids'].append(sig)
+        self.state.document.note_explicit_target(signode)
+        # Store the location of the option definition for later use in
+        # resolving cross-references
+        cached_options[sig] = self.env.docname
+
+
+def _indent(text):
+    if not text:
+        return text
+
+    padding = ' ' * 4
+    return padding + text
+
+
+def _format_validator_group_help(
+    validators: ty.Dict[str, base.ExtraSpecValidator],
+    summary: bool,
+):
+    """Generate reStructuredText snippets for a group of validators."""
+    for validator in validators.values():
+        for line in _format_validator_help(validator, summary):
+            yield line
+
+
+def _format_validator_help(
+    validator: base.ExtraSpecValidator,
+    summary: bool,
+):
+    """Generate reStucturedText snippets for the provided validator.
+
+    :param validator: A validator to document.
+    :type validator: nova.api.validation.extra_specs.base.ExtraSpecValidator
+    """
+    yield f'.. nova:extra-spec:: {validator.name}'
+    yield ''
+
+    # NOTE(stephenfin): We don't print the pattern, if present, since it's too
+    # internal. Instead, the description should provide this information in a
+    # human-readable format
+    yield _indent(f':Type: {validator.value["type"].__name__}')
+
+    if validator.value.get('min') is not None:
+        yield _indent(f':Min: {validator.value["min"]}')
+
+    if validator.value.get('max') is not None:
+        yield _indent(f':Max: {validator.value["max"]}')
+
+    yield ''
+
+    if not summary:
+        for line in validator.description.splitlines():
+            yield _indent(line)
+
+        yield ''
+
+    if validator.deprecated:
+        yield _indent('.. warning::')
+        yield _indent(
+            'This extra spec has been deprecated and should not be used.'
+        )
+
+    yield ''
+
+
+class ExtraSpecGroupDirective(rst.Directive):
+    """Document extra specs belonging to the specified group.
+
+    Accepts one optional argument - the extra spec group - and one option -
+    whether to show a summary view only (omit descriptions). Example::
+
+        .. extra-specs:: hw_rng
+           :summary:
+    """
+
+    required_arguments = 0
+    optional_arguments = 1
+    option_spec = {
+        'summary': directives.flag,
+    }
+    has_content = False
+
+    def run(self):
+        result = statemachine.ViewList()
+        source_name = self.state.document.current_source
+
+        group = self.arguments[0] if self.arguments else None
+        summary = self.options.get('summary', False)
+
+        if group:
+            group_validators = {
+                n.split(':', 1)[1]: v for n, v in validators.VALIDATORS.items()
+                if ':' in n and n.split(':', 1)[0].split('{')[0] == group
+            }
+        else:
+            group_validators = {
+                n: v for n, v in validators.VALIDATORS.items()
+                if ':' not in n
+            }
+
+        if not group_validators:
+            LOG.warning("No validators found for group '%s'", group or '')
+
+        for count, line in enumerate(
+            _format_validator_group_help(group_validators, summary)
+        ):
+            result.append(line, source_name, count)
+            LOG.debug('%5d%s%s', count, ' ' if line else '', line)
+
+        node = nodes.section()
+        node.document = self.state.document
+
+        sphinx_nodes.nested_parse_with_titles(self.state, result, node)
+
+        return node.children
+
+
+class NovaDomain(domains.Domain):
+    """nova domain."""
+    name = 'nova'
+    label = 'nova'
+    object_types = {
+        'configoption': domains.ObjType(
+            'extra spec', 'spec',
+        ),
+    }
+    directives = {
+        'extra-spec': ExtraSpecDirective,
+    }
+    roles = {
+        'extra-spec': ExtraSpecXRefRole(),
+    }
+    initial_data = {
+        'extra_specs': {},
+    }
+
+    def resolve_xref(
+        self, env, fromdocname, builder, typ, target, node, contnode,
+    ):
+        """Resolve cross-references"""
+        if typ == 'option':
+            return sphinx_nodes.make_refnode(
+                builder,
+                fromdocname,
+                env.domaindata['nova']['extra_specs'][target],
+                target,
+                contnode,
+                target,
+            )
+        return None
+
+
+def setup(app):
+    app.add_domain(NovaDomain)
+    app.add_directive('extra-specs', ExtraSpecGroupDirective)

doc/source/conf.py

@@ -41,6 +41,7 @@ extensions = [
     'oslo_policy.sphinxext',
     'ext.versioned_notifications',
     'ext.feature_matrix',
+    'ext.extra_specs',
     'sphinxcontrib.actdiag',
     'sphinxcontrib.seqdiag',
     'sphinxcontrib.rsvgconverter',
@@ -148,6 +149,7 @@ latex_use_xindy = False
 openstack_projects = [
     'ceilometer',
     'cinder',
+    'cyborg',
     'glance',
     'horizon',
     'ironic',

doc/source/configuration/extra-specs.rst

@@ -0,0 +1,212 @@
+===========
+Extra Specs
+===========
+
+The following is an overview of all extra specs recognized by nova in its
+default configuration.
+
+.. note::
+
+    Other services and virt drivers may provide additional extra specs not
+    listed here. In addition, it is possible to register your own extra specs.
+    For more information on the latter, refer to :doc:`/user/filter-scheduler`.
+
+Placement
+---------
+
+The following extra specs are used during scheduling to modify the request sent
+to placement.
+
+``resources``
+~~~~~~~~~~~~~
+
+The following extra specs are used to request an amount of the specified
+resource from placement when scheduling. All extra specs expect an integer
+value.
+
+.. note::
+
+   Not all of the resource types listed below are supported by all virt
+   drivers.
+
+.. extra-specs:: resources
+   :summary:
+
+``trait``
+~~~~~~~~~
+
+The following extra specs are used to request a specified trait from placement
+when scheduling. All extra specs expect one of the following values:
+
+- ``required``
+- ``forbidden``
+
+.. note::
+
+   Not all of the traits listed below are supported by all virt drivers.
+
+.. extra-specs:: trait
+   :summary:
+
+Scheduler Filters
+-----------------
+
+The following extra specs are specific to various in-tree scheduler filters.
+
+``aggregate_instance_extra_specs``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following extra specs are used to specify metadata that must be present on
+the aggregate of a host. If this metadata is not present or does not match the
+expected value, the aggregate and all hosts within in will be rejected.
+
+Requires the ``AggregateInstanceExtraSpecsFilter`` scheduler filter.
+
+.. extra-specs:: aggregate_instance_extra_specs
+
+``capabilities``
+~~~~~~~~~~~~~~~~
+
+The following extra specs are used to specify a host capability that must be
+provided by the host compute service. If this capability is not present or does
+not match the expected value, the host will be rejected.
+
+Requires the ``ComputeCapabilitiesFilter`` scheduler filter.
+
+All extra specs expect similar types of values:
+
+* ``=`` (equal to or greater than as a number; same as vcpus case)
+* ``==`` (equal to as a number)
+* ``!=`` (not equal to as a number)
+* ``>=`` (greater than or equal to as a number)
+* ``<=`` (less than or equal to as a number)
+* ``s==`` (equal to as a string)
+* ``s!=`` (not equal to as a string)
+* ``s>=`` (greater than or equal to as a string)
+* ``s>`` (greater than as a string)
+* ``s<=`` (less than or equal to as a string)
+* ``s<`` (less than as a string)
+* ``<in>`` (substring)
+* ``<all-in>`` (all elements contained in collection)
+* ``<or>`` (find one of these)
+* A specific value, e.g. ``true``, ``123``, ``testing``
+
+Examples are: ``>= 5``, ``s== 2.1.0``, ``<in> gcc``, ``<all-in> aes mmx``, and
+``<or> fpu <or> gpu``
+
+.. note::
+
+   Not all operators will apply to all types of values. For example, the ``==``
+   operator should not be used for a string value - use ``s==`` instead.
+
+.. extra-specs:: capabilities
+   :summary:
+
+Virt driver
+-----------
+
+The following extra specs are used as hints to configure internals of a
+instance, from the bus used for paravirtualized devices to the amount of a
+physical device to passthrough to the instance. Most of these are virt
+driver-specific.
+
+``quota``
+~~~~~~~~~
+
+The following extra specs are used to configure quotas for various
+paravirtualized devices.
+
+They are only supported by the libvirt virt driver.
+
+.. extra-specs:: quota
+
+``accel``
+~~~~~~~~~
+
+The following extra specs are used to configure attachment of various
+accelerators to an instance. For more information, refer to :cyborg-doc:`the
+Cyborg documentation <>`.
+
+They are only supported by the libvirt virt driver.
+
+.. extra-specs:: accel
+
+``pci_passthrough``
+~~~~~~~~~~~~~~~~~~~
+
+The following extra specs are used to configure passthrough of a host PCI
+device to an instance. This requires prior host configuration. For more
+information, refer to :doc:`/admin/pci-passthrough`.
+
+They are only supported by the libvirt virt driver.
+
+.. extra-specs:: pci_passthrough
+
+``hw``
+~~~~~~
+
+The following extra specs are used to configure various attributes of
+instances. Some of the extra specs act as feature flags, while others tweak for
+example the guest-visible CPU topology of the instance.
+
+Except where otherwise stated, they are only supported by the libvirt virt
+driver.
+
+.. extra-specs:: hw
+
+``hw_rng``
+~~~~~~~~~~
+
+The following extra specs are used to configure a random number generator for
+an instance.
+
+They are only supported by the libvirt virt driver.
+
+.. extra-specs:: hw_rng
+
+``hw_video``
+~~~~~~~~~~~~
+
+The following extra specs are used to configure attributes of the default guest
+video device.
+
+They are only supported by the libvirt virt driver.
+
+.. extra-specs:: hw_video
+
+``os``
+~~~~~~
+
+The following extra specs are used to configure various attributes of
+instances when using the HyperV virt driver.
+
+They are only supported by the HyperV virt driver.
+
+.. extra-specs:: os
+
+``powervm``
+~~~~~~~~~~~
+
+The following extra specs are used to configure various attributes of
+instances when using the PowerVM virt driver.
+
+They are only supported by the PowerVM virt driver.
+
+.. extra-specs:: powervm
+
+``vmware``
+~~~~~~~~~~
+
+The following extra specs are used to configure various attributes of
+instances when using the VMWare virt driver.
+
+They are only supported by the VMWare virt driver.
+
+.. extra-specs:: vmware
+
+Others (uncategorized)
+----------------------
+
+The following extra specs are not part of a group.
+
+.. extra-specs::

doc/source/configuration/index.rst

@@ -75,3 +75,19 @@ permissions on REST API actions.
       :hidden:
 
       sample-policy
+
+Extra Specs
+-----------
+
+Nova uses *flavor extra specs* as a way to provide additional information to
+instances beyond basic information like amount of RAM or disk. This information
+can range from hints for the scheduler to hypervisor-specific configuration
+instructions for the instance.
+
+* :doc:`Extra Spec Reference <extra-specs>`: A complete reference for all extra
+  specs currently recognized and supported by nova.
+
+.. toctree::
+   :hidden:
+
+   extra-specs

nova/api/validation/extra_specs/capabilities.py

@@ -18,17 +18,8 @@ These are used by the ``ComputeCapabilitiesFilter`` scheduler filter. Note that
 we explicitly do not allow the unnamespaced variant of extra specs since this
 has been deprecated since Grizzly (commit 8ce8e4b6c0d). Users that insist on
 using these can disable extra spec validation.
-"""
 
-from nova.api.validation.extra_specs import base
+For all extra specs, the value can be one of the following:
-
-
-DESCRIPTION = """\
-Specify that the '{capability}' capability provided by the host compute service
-satisfy the provided filter value. Requires the ``ComputeCapabilitiesFilter``
-scheduler filter.
-
-The value can be one of the following:
 
 * ``=`` (equal to or greater than as a number; same as vcpus case)
 * ``==`` (equal to as a number)
@@ -50,6 +41,15 @@ Examples are: ``>= 5``, ``s== 2.1.0``, ``<in> gcc``, ``<all-in> aes mmx``, and
 ``<or> fpu <or> gpu``
 """
 
+from nova.api.validation.extra_specs import base
+
+
+DESCRIPTION = """\
+Specify that the '{capability}' capability provided by the host compute service
+satisfy the provided filter value. Requires the ``ComputeCapabilitiesFilter``
+scheduler filter.
+"""
+
 EXTRA_SPEC_VALIDATORS = []
 
 # non-nested capabilities (from 'nova.objects.compute_node.ComputeNode' and