diff --git a/doc/source/user/filter-scheduler.rst b/doc/source/user/filter-scheduler.rst index 4ce5122871f0..6cc0e893de0c 100644 --- a/doc/source/user/filter-scheduler.rst +++ b/doc/source/user/filter-scheduler.rst @@ -370,42 +370,98 @@ For further details about each of those objects and their corresponding attributes, refer to the codebase (at least by looking at the other filters code) or ask for help in the #openstack-nova IRC channel. -The module containing your custom filter(s) must be packaged and available in -the same environment that nova, or specifically the :program:`nova-scheduler` -service, is available in. As an example, consider the following sample package, -which is the `minimal structure`__ for a standard, setuptools-based Python -package: +In addition, if your custom filter uses non-standard extra specs, you must +register validators for these extra specs. Examples of validators can be found +in the ``nova.api.validation.extra_specs`` module. These should be registered +via the ``nova.api.extra_spec_validator`` `entrypoint`__. +The module containing your custom filter(s) must be packaged and available in +the same environment(s) that the nova controllers, or specifically the +:program:`nova-scheduler` and :program:`nova-api` services, are available in. +As an example, consider the following sample package, which is the `minimal +structure`__ for a standard, setuptools-based Python package: + +__ https://packaging.python.org/specifications/entry-points/ __ https://python-packaging.readthedocs.io/en/latest/minimal.html .. code-block:: none - myfilter/ - myfilter/ + acmefilter/ + acmefilter/ __init__.py + validators.py setup.py -The ``myfilter/myfilter/__init__.py`` could contain something like so: +Where ``__init__.py`` contains: .. code-block:: python + from oslo_log import log as logging from nova.scheduler import filters + LOG = logging.getLogger(__name__) - class MyFilter(filters.BaseHostFilter): + class AcmeFilter(filters.BaseHostFilter): def host_passes(self, host_state, spec_obj): - # do stuff here... + extra_spec = spec_obj.flavor.extra_specs.get('acme:foo') + LOG.info("Extra spec value was '%s'", extra_spec) + + # do meaningful stuff here... + return True +``validators.py`` contains: + +.. code-block:: python + + from nova.api.validation.extra_specs import base + + def register(): + validators = [ + base.ExtraSpecValidator( + name='acme:foo', + description='My custom extra spec.' + value={ + 'type': str, + 'enum': [ + 'bar', + 'baz', + ], + }, + ), + ] + + return validators + +``setup.py`` contains: + +.. code-block:: python + + from setuptools import setup + + setup( + name='acmefilter', + version='0.1', + description='My custom filter', + packages=[ + 'acmefilter' + ], + entry_points={ + 'nova.api.extra_spec_validators': [ + 'acme = acmefilter.validators', + ], + }, + ) + To enable this, you would set the following in :file:`nova.conf`: .. code-block:: ini [filter_scheduler] available_filters = nova.scheduler.filters.all_filters - available_filters = myfilter.MyFilter - enabled_filters = ComputeFilter,MyFilter + available_filters = acmefilter.AcmeFilter + enabled_filters = ComputeFilter,AcmeFilter .. note:: @@ -417,9 +473,9 @@ To enable this, you would set the following in :file:`nova.conf`: includes the filters shipped with nova. With these settings, nova will use the ``FilterScheduler`` for the scheduler -driver. All of the standard nova filters and the custom ``MyFilter`` filter are -available to the ``FilterScheduler``, but just the ``ComputeFilter`` and -``MyFilter`` will be used on each request. +driver. All of the standard nova filters and the custom ``AcmeFilter`` filter +are available to the ``FilterScheduler``, but just the ``ComputeFilter`` and +``AcmeFilter`` will be used on each request. Weights ------- diff --git a/lower-constraints.txt b/lower-constraints.txt index 2fb27f47e24f..c8beee8dc329 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -15,6 +15,7 @@ colorama==0.3.9 coverage==4.0 cryptography==2.7 cursive==0.2.1 +dataclasses==0.7 ddt==1.0.1 debtcollector==1.19.0 decorator==3.4.0 diff --git a/nova/api/openstack/compute/flavors_extraspecs.py b/nova/api/openstack/compute/flavors_extraspecs.py index a5c5c5e150d6..481a16efedcc 100644 --- a/nova/api/openstack/compute/flavors_extraspecs.py +++ b/nova/api/openstack/compute/flavors_extraspecs.py @@ -20,6 +20,7 @@ from nova.api.openstack import common from nova.api.openstack.compute.schemas import flavors_extraspecs from nova.api.openstack import wsgi from nova.api import validation +from nova.api.validation.extra_specs import validators from nova import exception from nova.i18n import _ from nova.policies import flavor_extra_specs as fes_policies @@ -28,22 +29,31 @@ from nova import utils class FlavorExtraSpecsController(wsgi.Controller): """The flavor extra specs API controller for the OpenStack API.""" + def _get_extra_specs(self, context, flavor_id): flavor = common.get_flavor(context, flavor_id) return dict(extra_specs=flavor.extra_specs) - # NOTE(gmann): Max length for numeric value is being checked - # explicitly as json schema cannot have max length check for numeric value - def _check_extra_specs_value(self, specs): - for value in specs.values(): - try: - if isinstance(value, (six.integer_types, float)): - value = six.text_type(value) + def _check_extra_specs_value(self, req, specs): + # TODO(stephenfin): Wire this up to check the API microversion + validation_supported = False + validation_mode = 'strict' + + for name, value in specs.items(): + # NOTE(gmann): Max length for numeric value is being checked + # explicitly as json schema cannot have max length check for + # numeric value + if isinstance(value, (six.integer_types, float)): + value = six.text_type(value) + try: utils.check_string_length(value, 'extra_specs value', max_length=255) - except exception.InvalidInput as error: - raise webob.exc.HTTPBadRequest( - explanation=error.format_message()) + except exception.InvalidInput as error: + raise webob.exc.HTTPBadRequest( + explanation=error.format_message()) + + if validation_supported: + validators.validate(name, value, validation_mode) @wsgi.expected_errors(404) def index(self, req, flavor_id): @@ -62,7 +72,7 @@ class FlavorExtraSpecsController(wsgi.Controller): context.can(fes_policies.POLICY_ROOT % 'create') specs = body['extra_specs'] - self._check_extra_specs_value(specs) + self._check_extra_specs_value(req, specs) flavor = common.get_flavor(context, flavor_id) try: flavor.extra_specs = dict(flavor.extra_specs, **specs) @@ -79,7 +89,7 @@ class FlavorExtraSpecsController(wsgi.Controller): context = req.environ['nova.context'] context.can(fes_policies.POLICY_ROOT % 'update') - self._check_extra_specs_value(body) + self._check_extra_specs_value(req, body) if id not in body: expl = _('Request body and URI mismatch') raise webob.exc.HTTPBadRequest(explanation=expl) diff --git a/nova/api/validation/extra_specs/__init__.py b/nova/api/validation/extra_specs/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/nova/api/validation/extra_specs/aggregate_instance_extra_specs.py b/nova/api/validation/extra_specs/aggregate_instance_extra_specs.py new file mode 100644 index 000000000000..0cf23b28ff6b --- /dev/null +++ b/nova/api/validation/extra_specs/aggregate_instance_extra_specs.py @@ -0,0 +1,72 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for (preferrably) ``aggregate_instance_extra_specs`` namespaced +extra specs. + +These are used by the ``AggregateInstanceExtraSpecsFilter`` scheduler filter. +Note that we explicitly do not support the unnamespaced variant of extra specs +since these have been deprecated since Havana (commit fbedf60a432). Users that +insist on using these can disable extra spec validation. +""" + +from nova.api.validation.extra_specs import base + + +DESCRIPTION = """\ +Specify metadata that must be present on the aggregate of a host. If this +metadata is not present, the host will be rejected. Requires the +``AggregateInstanceExtraSpecsFilter`` scheduler filter. + +The value can be one of the following: + +* ``=`` (equal to or greater than as a number; same as vcpus case) +* ``==`` (equal to as a number) +* ``!=`` (not equal to as a number) +* ``>=`` (greater than or equal to as a number) +* ``<=`` (less than or equal to as a number) +* ``s==`` (equal to as a string) +* ``s!=`` (not equal to as a string) +* ``s>=`` (greater than or equal to as a string) +* ``s>`` (greater than as a string) +* ``s<=`` (less than or equal to as a string) +* ``s<`` (less than as a string) +* ```` (substring) +* ```` (all elements contained in collection) +* ```` (find one of these) +* A specific value, e.g. ``true``, ``123``, ``testing`` +""" + +EXTRA_SPEC_VALIDATORS = [ + base.ExtraSpecValidator( + name='aggregate_instance_extra_specs:{key}', + description=DESCRIPTION, + parameters=[ + { + 'name': 'key', + 'description': 'The metadata key to match on', + 'pattern': r'.+', + }, + ], + value={ + # this is totally arbitary, since we need to support specific + # values + 'type': str, + }, + ), +] + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/api/validation/extra_specs/base.py b/nova/api/validation/extra_specs/base.py new file mode 100644 index 000000000000..2597070127ac --- /dev/null +++ b/nova/api/validation/extra_specs/base.py @@ -0,0 +1,120 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import dataclasses +import re +import typing as ty + +from oslo_utils import strutils + +from nova import exception + + +@dataclasses.dataclass +class ExtraSpecValidator: + name: str + description: str + value: ty.Dict[str, ty.Any] + deprecated: bool = False + parameters: ty.List[ty.Dict[str, ty.Any]] = dataclasses.field( + default_factory=list + ) + + name_regex: str = None + value_regex: str = None + + def __post_init__(self): + # generate a regex for the name + + name_regex = self.name + # replace the human-readable patterns with named regex groups; this + # will transform e.g. 'hw:numa_cpus.{id}' to 'hw:numa_cpus.(?P\d+)' + for param in self.parameters: + pattern = f'(?P<{param["name"]}>{param["pattern"]})' + name_regex = name_regex.replace(f'{{{param["name"]}}}', pattern) + + self.name_regex = name_regex + + # ...and do the same for the value, but only if we're using strings + + if self.value['type'] not in (int, str, bool): + raise ValueError( + f"Unsupported parameter type '{self.value['type']}'" + ) + + value_regex = None + if self.value['type'] == str and self.value.get('pattern'): + value_regex = self.value['pattern'] + + self.value_regex = value_regex + + def _validate_str(self, value): + if 'pattern' in self.value: + value_match = re.fullmatch(self.value_regex, value) + if not value_match: + raise exception.ValidationError( + f"Validation failed; '{value}' is not of the format " + f"'{self.value_regex}'." + ) + elif 'enum' in self.value: + if value not in self.value['enum']: + values = ', '.join(str(x) for x in self.value['enum']) + raise exception.ValidationError( + f"Validation failed; '{value}' is not one of: {values}." + ) + + def _validate_int(self, value): + try: + value = int(value) + except ValueError: + raise exception.ValidationError( + f"Validation failed; '{value}' is not a valid integer value." + ) + + if 'max' in self.value and self.value['max'] < value: + raise exception.ValidationError( + f"Validation failed; '{value}' is greater than the max value " + f"of '{self.value['max']}'." + ) + + if 'min' in self.value and self.value['min'] > value: + raise exception.ValidationError( + f"Validation failed; '{value}' is less than the min value " + f"of '{self.value['min']}'." + ) + + def _validate_bool(self, value): + try: + strutils.bool_from_string(value, strict=True) + except ValueError: + raise exception.ValidationError( + f"Validation failed; '{value}' is not a valid boolean-like " + f"value." + ) + + def validate(self, name, value): + name_match = re.fullmatch(self.name_regex, name) + if not name_match: + # NOTE(stephenfin): This is mainly here for testing purposes + raise exception.ValidationError( + f"Validation failed; expected a name of format '{self.name}' " + f"but got '{name}'." + ) + + if self.value['type'] == int: + self._validate_int(value) + elif self.value['type'] == bool: + self._validate_bool(value) + else: # str + self._validate_str(value) diff --git a/nova/api/validation/extra_specs/capabilities.py b/nova/api/validation/extra_specs/capabilities.py new file mode 100644 index 000000000000..58ffff0813d9 --- /dev/null +++ b/nova/api/validation/extra_specs/capabilities.py @@ -0,0 +1,112 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for (preferrably) ``capabilities`` namespaced extra specs. + +These are used by the ``ComputeCapabilitiesFilter`` scheduler filter. Note that +we explicitly do not allow the unnamespaced variant of extra specs since this +has been deprecated since Grizzly (commit 8ce8e4b6c0d). Users that insist on +using these can disable extra spec validation. +""" + +from nova.api.validation.extra_specs import base + + +DESCRIPTION = """\ +Specify that the '{capability}' capability provided by the host compute service +satisfy the provided filter value. Requires the ``ComputeCapabilitiesFilter`` +scheduler filter. + +The value can be one of the following: + +* ``=`` (equal to or greater than as a number; same as vcpus case) +* ``==`` (equal to as a number) +* ``!=`` (not equal to as a number) +* ``>=`` (greater than or equal to as a number) +* ``<=`` (less than or equal to as a number) +* ``s==`` (equal to as a string) +* ``s!=`` (not equal to as a string) +* ``s>=`` (greater than or equal to as a string) +* ``s>`` (greater than as a string) +* ``s<=`` (less than or equal to as a string) +* ``s<`` (less than as a string) +* ```` (substring) +* ```` (all elements contained in collection) +* ```` (find one of these) +* A specific value, e.g. ``true``, ``123``, ``testing`` + +Examples are: ``>= 5``, ``s== 2.1.0``, `` gcc``, `` aes mmx``, and +`` fpu gpu`` +""" + +EXTRA_SPEC_VALIDATORS = [] + +# non-nested capabilities (from 'nova.objects.compute_node.ComputeNode' and +# nova.scheduler.host_manager.HostState') + +for capability in ( + 'id', 'uuid', 'service_id', 'host', 'vcpus', 'memory_mb', 'local_gb', + 'vcpus_used', 'memory_mb_used', 'local_gb_used', + 'hypervisor_type', 'hypervisor_version', 'hypervisor_hostname', + 'free_ram_mb', 'free_disk_gb', 'current_workload', 'running_vms', + 'disk_available_least', 'host_ip', 'mapped', + 'cpu_allocation_ratio', 'ram_allocation_ratio', 'disk_allocation_ratio', +) + ( + 'total_usable_ram_mb', 'total_usable_disk_gb', 'disk_mb_used', + 'free_disk_mb', 'vcpus_total', 'vcpus_used', 'num_instances', + 'num_io_ops', 'failed_builds', 'aggregates', 'cell_uuid', 'updated', +): + EXTRA_SPEC_VALIDATORS.append( + base.ExtraSpecValidator( + name=f'capabilities:{capability}', + description=DESCRIPTION.format(capability=capability), + value={ + # this is totally arbitary, since we need to support specific + # values + 'type': str, + }, + ), + ) + + +# nested capabilities (from 'nova.objects.compute_node.ComputeNode' and +# nova.scheduler.host_manager.HostState') + +for capability in ( + 'cpu_info', 'metrics', 'stats', 'numa_topology', 'supported_hv_specs', + 'pci_device_pools', +) + ( + 'nodename', 'pci_stats', 'supported_instances', 'limits', 'instances', +): + EXTRA_SPEC_VALIDATORS.extend([ + base.ExtraSpecValidator( + name=f'capabilities:{capability}{{filter}}', + description=DESCRIPTION.format(capability=capability), + parameters=[ + { + 'name': 'filter', + # this is optional, but if it's present it must be preceded + # by ':' + 'pattern': r'(:\w+)*', + } + ], + value={ + 'type': str, + }, + ), + ]) + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/api/validation/extra_specs/hw.py b/nova/api/validation/extra_specs/hw.py new file mode 100644 index 000000000000..f5e8b7e7e795 --- /dev/null +++ b/nova/api/validation/extra_specs/hw.py @@ -0,0 +1,370 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for ``hw`` namespaced extra specs.""" + +from nova.api.validation.extra_specs import base + + +realtime_validators = [ + base.ExtraSpecValidator( + name='hw:cpu_realtime', + description=( + 'Determine whether realtime mode should be enabled for the ' + 'instance or not. Only supported by the libvirt driver.' + ), + value={ + 'type': bool, + 'description': 'Whether to enable realtime priority.', + }, + ), + base.ExtraSpecValidator( + name='hw:cpu_realtime_mask', + description=( + 'A exclusion mask of CPUs that should not be enabled for realtime.' + ), + value={ + 'type': str, + # NOTE(stephenfin): Yes, these things *have* to start with '^' + 'pattern': r'\^\d+((-\d+)?(,\^?\d+(-\d+)?)?)*', + }, + ), +] + +cpu_policy_validators = [ + base.ExtraSpecValidator( + name='hw:cpu_policy', + description=( + 'The policy to apply when determining what host CPUs the guest ' + 'CPUs can run on. If ``shared`` (default), guest CPUs can be ' + 'overallocated but cannot float across host cores. If ' + '``dedicated``, guest CPUs cannot be overallocated but are ' + 'individually pinned to their own host core.' + ), + value={ + 'type': str, + 'description': 'The CPU policy.', + 'enum': [ + 'dedicated', + 'shared' + ], + }, + ), + base.ExtraSpecValidator( + name='hw:cpu_thread_policy', + description=( + 'The policy to apply when determining whether the destination ' + 'host can have hardware threads enabled or not. If ``prefer`` ' + '(default), hosts with hardware threads will be preferred. If ' + '``require``, hosts with hardware threads will be required. If ' + '``isolate``, hosts with hardware threads will be forbidden.' + ), + value={ + 'type': str, + 'description': 'The CPU thread policy.', + 'enum': [ + 'prefer', + 'isolate', + 'require', + ], + }, + ), + base.ExtraSpecValidator( + name='hw:emulator_threads_policy', + description=( + 'The policy to apply when determining whether emulator threads ' + 'should be offloaded to a separate isolated core or to a pool ' + 'of shared cores. If ``share``, emulator overhead threads will ' + 'be offloaded to a pool of shared cores. If ``isolate``, ' + 'emulator overhead threads will be offloaded to their own core.' + ), + value={ + 'type': str, + 'description': 'The emulator thread policy.', + 'enum': [ + 'isolate', + 'share', + ], + }, + ), +] + +hugepage_validators = [ + base.ExtraSpecValidator( + name='hw:mem_page_size', + description=( + 'The size of memory pages to allocate to the guest with. Can be ' + 'one of the three alias - ``large``, ``small`` or ``any``, - or ' + 'an actual size. Only supported by the libvirt virt driver.' + ), + value={ + 'type': str, + 'description': 'The size of memory page to allocate', + 'pattern': r'(large|small|any|\d+([kKMGT]i?)?(b|bit|B)?)', + }, + ), +] + +numa_validators = [ + base.ExtraSpecValidator( + name='hw:numa_nodes', + description=( + 'The number of virtual NUMA nodes to allocate to configure the ' + 'guest with. Each virtual NUMA node will be mapped to a unique ' + 'host NUMA node. Only supported by the libvirt virt driver.' + ), + value={ + 'type': int, + 'description': 'The number of virtual NUMA nodes to allocate', + 'min': 1, + }, + ), + base.ExtraSpecValidator( + name='hw:numa_cpus.{id}', + description=( + 'A mapping of **guest** CPUs to the **guest** NUMA node ' + 'identified by ``{id}``. This can be used to provide asymmetric ' + 'CPU-NUMA allocation and is necessary where the number of guest ' + 'NUMA nodes is not a factor of the number of guest CPUs.' + ), + parameters=[ + { + 'name': 'id', + 'pattern': r'\d+', # positive integers + 'description': 'The ID of the **guest** NUMA node.', + }, + ], + value={ + 'type': str, + 'description': ( + 'The guest CPUs, in the form of a CPU map, to allocate to the ' + 'guest NUMA node identified by ``{id}``.' + ), + 'pattern': r'\^?\d+((-\d+)?(,\^?\d+(-\d+)?)?)*', + }, + ), + base.ExtraSpecValidator( + name='hw:numa_mem.{id}', + description=( + 'A mapping of **guest** memory to the **guest** NUMA node ' + 'identified by ``{id}``. This can be used to provide asymmetric ' + 'memory-NUMA allocation and is necessary where the number of ' + 'guest NUMA nodes is not a factor of the total guest memory.' + ), + parameters=[ + { + 'name': 'id', + 'pattern': r'\d+', # positive integers + 'description': 'The ID of the **guest** NUMA node.', + }, + ], + value={ + 'type': int, + 'description': ( + 'The guest memory, in MB, to allocate to the guest NUMA node ' + 'identified by ``{id}``.' + ), + 'min': 1, + }, + ), + base.ExtraSpecValidator( + name='hw:pci_numa_affinity_policy', + description=( + 'The NUMA affinity policy of any PCI passthrough devices or ' + 'SR-IOV network interfaces attached to the instance.' + ), + value={ + 'type': str, + 'description': 'The PCI NUMA affinity policy', + 'enum': [ + 'required', + 'preferred', + 'legacy', + ], + }, + ), +] + +cpu_topology_validators = [ + base.ExtraSpecValidator( + name='hw:cpu_sockets', + description=( + 'The number of virtual CPU threads to emulate in the guest ' + 'CPU topology.' + ), + value={ + 'type': int, + 'description': 'A number of vurtla CPU sockets', + 'min': 1, + }, + ), + base.ExtraSpecValidator( + name='hw:cpu_cores', + description=( + 'The number of virtual CPU cores to emulate per socket in the ' + 'guest CPU topology.' + ), + value={ + 'type': int, + 'description': 'A number of virtual CPU cores', + 'min': 1, + }, + ), + base.ExtraSpecValidator( + name='hw:cpu_threads', + description=( + 'The number of virtual CPU threads to emulate per core in the ' + 'guest CPU topology.' + ), + value={ + 'type': int, + 'description': 'A number of virtual CPU threads', + 'min': 1, + }, + ), + base.ExtraSpecValidator( + name='hw:max_cpu_sockets', + description=( + 'The max number of virtual CPU threads to emulate in the ' + 'guest CPU topology. This is used to limit the topologies that ' + 'can be requested by an image and will be used to validate the ' + '``hw_cpu_sockets`` image metadata property.' + ), + value={ + 'type': int, + 'description': 'A number of virtual CPU sockets', + 'min': 1, + }, + ), + base.ExtraSpecValidator( + name='hw:max_cpu_cores', + description=( + 'The max number of virtual CPU cores to emulate per socket in the ' + 'guest CPU topology. This is used to limit the topologies that ' + 'can be requested by an image and will be used to validate the ' + '``hw_cpu_cores`` image metadata property.' + ), + value={ + 'type': int, + 'description': 'A number of virtual CPU cores', + 'min': 1, + }, + ), + base.ExtraSpecValidator( + name='hw:max_cpu_threads', + description=( + 'The max number of virtual CPU threads to emulate per core in the ' + 'guest CPU topology. This is used to limit the topologies that ' + 'can be requested by an image and will be used to validate the ' + '``hw_cpu_threads`` image metadata property.' + ), + value={ + 'type': int, + 'description': 'A number of virtual CPU threads', + 'min': 1, + }, + ), +] + +feature_flag_validators = [ + # TODO(stephenfin): Consider deprecating and moving this to the 'os:' + # namespace + base.ExtraSpecValidator( + name='hw:boot_menu', + description=( + 'Whether to show a boot menu when booting the guest.' + ), + value={ + 'type': bool, + 'description': 'Whether to enable the boot menu', + }, + ), + base.ExtraSpecValidator( + name='hw:mem_encryption', + description=( + 'Whether to enable memory encryption for the guest. Only ' + 'supported by the libvirt driver on hosts with AMD SEV support.' + ), + value={ + 'type': bool, + 'description': 'Whether to enable memory encryption', + }, + ), + base.ExtraSpecValidator( + name='hw:pmem', + description=( + 'A comma-separated list of ``$LABEL``\\ s defined in config for ' + 'vPMEM devices.' + ), + value={ + 'type': str, + 'description': ( + 'A comma-separated list of valid resource class names.' + ), + 'pattern': '([a-zA-Z0-9_]+(,)?)+', + }, + ), + base.ExtraSpecValidator( + name='hw:pmu', + description=( + 'Whether to enable the Performance Monitory Unit (PMU) for the ' + 'guest. Only supported by the libvirt driver.' + ), + value={ + 'type': bool, + 'description': 'Whether to enable the PMU', + }, + ), + base.ExtraSpecValidator( + name='hw:serial_port_count', + description=( + 'The number of serial ports to allocate to the guest. Only ' + 'supported by the libvirt virt driver.' + ), + value={ + 'type': int, + 'min': 0, + 'description': 'The number of serial ports to allocate', + }, + ), + base.ExtraSpecValidator( + name='hw:watchdog_action', + description=( + 'The action to take when the watchdog timer is kicked. Only ' + 'supported by the libvirt virt driver.' + ), + value={ + 'type': str, + 'description': 'The action to take', + 'enum': [ + 'none', + 'pause', + 'poweroff', + 'reset', + 'disabled', + ], + }, + ), +] + + +def register(): + return ( + realtime_validators + + cpu_policy_validators + + hugepage_validators + + numa_validators + + cpu_topology_validators + + feature_flag_validators + ) diff --git a/nova/api/validation/extra_specs/hw_rng.py b/nova/api/validation/extra_specs/hw_rng.py new file mode 100644 index 000000000000..4d516ef49a63 --- /dev/null +++ b/nova/api/validation/extra_specs/hw_rng.py @@ -0,0 +1,57 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for ``hw_rng`` namespaced extra specs.""" + +from nova.api.validation.extra_specs import base + + +# TODO(stephenfin): Move these to the 'hw:' namespace +EXTRA_SPEC_VALIDATORS = [ + base.ExtraSpecValidator( + name='hw_rng:allowed', + description=( + 'Whether to disable configuration of a random number generator ' + 'in their image. Before 21.0.0 (Ussuri), random number generators ' + 'were not enabled by default so this was used to determine ' + 'whether to **enable** configuration.' + ), + value={ + 'type': bool, + }, + ), + base.ExtraSpecValidator( + name='hw_rng:rate_bytes', + description=( + 'The allowed amount of bytes for the guest to read from the ' + 'host\'s entropy per period.' + ), + value={ + 'type': int, + 'min': 0, + }, + ), + base.ExtraSpecValidator( + name='hw_rng:rate_period', + description='The duration of a read period in seconds.', + value={ + 'type': int, + 'min': 0, + }, + ), +] + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/api/validation/extra_specs/hw_video.py b/nova/api/validation/extra_specs/hw_video.py new file mode 100644 index 000000000000..dc9aa0fe6657 --- /dev/null +++ b/nova/api/validation/extra_specs/hw_video.py @@ -0,0 +1,39 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for ``hw_video`` namespaced extra specs.""" + +from nova.api.validation.extra_specs import base + + +# TODO(stephenfin): Move these to the 'hw:' namespace +EXTRA_SPEC_VALIDATORS = [ + base.ExtraSpecValidator( + name='hw_video:ram_max_mb', + description=( + 'The maximum amount of memory the user can request using the ' + '``hw_video_ram`` image metadata property, which represents the ' + 'video memory that the guest OS will see. This has no effect for ' + 'vGPUs.' + ), + value={ + 'type': int, + 'min': 0, + }, + ), +] + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/api/validation/extra_specs/null.py b/nova/api/validation/extra_specs/null.py new file mode 100644 index 000000000000..7270c0e600ca --- /dev/null +++ b/nova/api/validation/extra_specs/null.py @@ -0,0 +1,51 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for non-namespaced extra specs.""" + +from nova.api.validation.extra_specs import base + + +# TODO(stephenfin): These should be moved to a namespace +EXTRA_SPEC_VALIDATORS = [ + base.ExtraSpecValidator( + name='hide_hypervisor_id', + description=( + 'Determine whether the hypervisor ID should be hidden from the ' + 'guest. Only supported by the libvirt driver.' + ), + value={ + 'type': bool, + 'description': 'Whether to hide the hypervisor ID.', + }, + ), + base.ExtraSpecValidator( + name='group_policy', + description=( + 'The group policy to apply when using the granular resource ' + 'request syntax.' + ), + value={ + 'type': str, + 'enum': [ + 'isolate', + 'none', + ], + }, + ), +] + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/api/validation/extra_specs/os.py b/nova/api/validation/extra_specs/os.py new file mode 100644 index 000000000000..1fc9c76ad443 --- /dev/null +++ b/nova/api/validation/extra_specs/os.py @@ -0,0 +1,95 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for ``os`` namespaced extra specs.""" + +from nova.api.validation.extra_specs import base + + +# TODO(stephenfin): Most of these belong in the 'hw:' or 'hyperv:' namespace +# and should be moved. +EXTRA_SPEC_VALIDATORS = [ + base.ExtraSpecValidator( + name='os:secure_boot', + description=( + 'Determine whether secure boot is enabled or not. Currently only ' + 'supported by the HyperV driver.' + ), + value={ + 'type': str, + 'description': 'Whether secure boot is required or not', + 'enum': [ + 'disabled', + 'required', + ], + }, + ), + base.ExtraSpecValidator( + name='os:resolution', + description=( + 'Guest VM screen resolution size. Only supported by the HyperV ' + 'driver.' + ), + value={ + 'type': str, + 'description': 'The chosen resolution', + 'enum': [ + '1024x768', + '1280x1024', + '1600x1200', + '1920x1200', + '2560x1600', + '3840x2160', + ], + }, + ), + base.ExtraSpecValidator( + name='os:monitors', + description=( + 'Guest VM number of monitors. Only supported by the HyperV driver.' + ), + value={ + 'type': int, + 'description': 'The number of monitors enabled', + 'min': 1, + 'max': 8, + }, + ), + # TODO(stephenfin): Consider merging this with the 'hw_video_ram' image + # metadata property or adding a 'hw:video_ram' extra spec that works for + # both Hyper-V and libvirt. + base.ExtraSpecValidator( + name='os:vram', + description=( + 'Guest VM VRAM amount. Only supported by the HyperV driver.' + ), + # NOTE(stephenfin): This is really an int, but because there's a + # limited range of options we treat it as a string + value={ + 'type': str, + 'description': 'Amount of VRAM to allocate to instance', + 'enum': [ + '64', + '128', + '256', + '512', + '1024', + ], + }, + ), +] + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/api/validation/extra_specs/pci_passthrough.py b/nova/api/validation/extra_specs/pci_passthrough.py new file mode 100644 index 000000000000..b594e170e9d0 --- /dev/null +++ b/nova/api/validation/extra_specs/pci_passthrough.py @@ -0,0 +1,38 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for ``pci_passthrough`` namespaced extra specs.""" + +from nova.api.validation.extra_specs import base + + +EXTRA_SPEC_VALIDATORS = [ + base.ExtraSpecValidator( + name='pci_passthrough:alias', + description=( + 'Specify the number of ``$alias`` PCI device(s) to attach to the ' + 'instance. Must be of format ``$alias:$number``. Use commas to ' + 'specify multiple values.' + ), + value={ + 'type': str, + # one or more comma-separated '$alias:$num' values + 'pattern': r'[^:]+:\d+(?:\s*,\s*[^:]+:\d+)*', + }, + ), +] + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/api/validation/extra_specs/powervm.py b/nova/api/validation/extra_specs/powervm.py new file mode 100644 index 000000000000..58ef79377761 --- /dev/null +++ b/nova/api/validation/extra_specs/powervm.py @@ -0,0 +1,271 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for ``powervm`` namespaced extra specs. + +These were all taken from the IBM documentation. + +https://www.ibm.com/support/knowledgecenter/SSXK2N_1.4.4/com.ibm.powervc.standard.help.doc/powervc_pg_flavorsextraspecs_hmc.html +""" + +from nova.api.validation.extra_specs import base + + +# TODO(stephenfin): A lot of these seem to overlap with existing 'hw:' extra +# specs and could be deprecated in favour of those. +EXTRA_SPEC_VALIDATORS = [ + base.ExtraSpecValidator( + name='powervm:min_mem', + description=( + 'Minimum memory (MB). If you do not specify the value, the value ' + 'is defaulted to the value for ``memory_mb``.' + ), + value={ + 'type': int, + 'min': 256, + 'description': 'Integer >=256 divisible by LMB size of the target', + }, + ), + base.ExtraSpecValidator( + name='powervm:max_mem', + description=( + 'Maximum memory (MB). If you do not specify the value, the value ' + 'is defaulted to the value for ``memory_mb``.' + ), + value={ + 'type': int, + 'min': 256, + 'description': 'Integer >=256 divisible by LMB size of the target', + }, + ), + base.ExtraSpecValidator( + name='powervm:min_vcpu', + description=( + 'Minimum virtual processors. Minimum resource that is required ' + 'for LPAR to boot is 1. The maximum value can be equal to the ' + 'value, which is set to vCPUs. If you specify the value of the ' + 'attribute, you must also specify value of powervm:max_vcpu. ' + 'Defaults to value set for vCPUs.' + ), + value={ + 'type': int, + 'min': 1, + }, + ), + base.ExtraSpecValidator( + name='powervm:max_vcpu', + description=( + 'Minimum virtual processors. Minimum resource that is required ' + 'for LPAR to boot is 1. The maximum value can be equal to the ' + 'value, which is set to vCPUs. If you specify the value of the ' + 'attribute, you must also specify value of powervm:max_vcpu. ' + 'Defaults to value set for vCPUs.' + ), + value={ + 'type': int, + 'min': 1, + }, + ), + base.ExtraSpecValidator( + name='powervm:proc_units', + description=( + 'The wanted ``proc_units``. The value for the attribute cannot be ' + 'less than 1/10 of the value that is specified for Virtual ' + 'CPUs (vCPUs) for hosts with firmware level 7.5 or earlier and ' + '1/20 of the value that is specified for vCPUs for hosts with ' + 'firmware level 7.6 or later. If the value is not specified ' + 'during deployment, it is defaulted to vCPUs * 0.5.' + ), + value={ + 'type': str, + 'pattern': r'\d+\.\d+', + 'description': ( + 'Float (divisible by 0.1 for hosts with firmware level 7.5 or ' + 'earlier and 0.05 for hosts with firmware level 7.6 or later)' + ), + }, + ), + base.ExtraSpecValidator( + name='powervm:min_proc_units', + description=( + 'Minimum ``proc_units``. The minimum value for the attribute is ' + '0.1 for hosts with firmware level 7.5 or earlier and 0.05 for ' + 'hosts with firmware level 7.6 or later. The maximum value must ' + 'be equal to the maximum value of ``powervm:proc_units``. If you ' + 'specify the attribute, you must also specify ' + '``powervm:proc_units``, ``powervm:max_proc_units``, ' + '``powervm:min_vcpu``, `powervm:max_vcpu``, and ' + '``powervm:dedicated_proc``. Set the ``powervm:dedicated_proc`` ' + 'to false.' + '\n' + 'The value for the attribute cannot be less than 1/10 of the ' + 'value that is specified for powervm:min_vcpu for hosts with ' + 'firmware level 7.5 or earlier and 1/20 of the value that is ' + 'specified for ``powervm:min_vcpu`` for hosts with firmware ' + 'level 7.6 or later. If you do not specify the value of the ' + 'attribute during deployment, it is defaulted to equal the value ' + 'of ``powervm:proc_units``.' + ), + value={ + 'type': str, + 'pattern': r'\d+\.\d+', + 'description': ( + 'Float (divisible by 0.1 for hosts with firmware level 7.5 or ' + 'earlier and 0.05 for hosts with firmware level 7.6 or later)' + ), + }, + ), + base.ExtraSpecValidator( + name='powervm:max_proc_units', + description=( + 'Maximum ``proc_units``. The minimum value can be equal to `` ' + '``powervm:proc_units``. The maximum value for the attribute ' + 'cannot be more than the value of the host for maximum allowed ' + 'processors per partition. If you specify this attribute, you ' + 'must also specify ``powervm:proc_units``, ' + '``powervm:min_proc_units``, ``powervm:min_vcpu``, ' + '``powervm:max_vcpu``, and ``powervm:dedicated_proc``. Set the ' + '``powervm:dedicated_proc`` to false.' + '\n' + 'The value for the attribute cannot be less than 1/10 of the ' + 'value that is specified for powervm:max_vcpu for hosts with ' + 'firmware level 7.5 or earlier and 1/20 of the value that is ' + 'specified for ``powervm:max_vcpu`` for hosts with firmware ' + 'level 7.6 or later. If you do not specify the value of the ' + 'attribute during deployment, the value is defaulted to equal the ' + 'value of ``powervm:proc_units``.' + ), + value={ + 'type': str, + 'pattern': r'\d+\.\d+', + 'description': ( + 'Float (divisible by 0.1 for hosts with firmware level 7.5 or ' + 'earlier and 0.05 for hosts with firmware level 7.6 or later)' + ), + }, + ), + base.ExtraSpecValidator( + name='powervm:dedicated_proc', + description=( + 'Use dedicated processors. The attribute defaults to false.' + ), + value={ + 'type': bool, + }, + ), + base.ExtraSpecValidator( + name='powervm:shared_weight', + description=( + 'Shared processor weight. When ``powervm:dedicated_proc`` is set ' + 'to true and ``powervm:uncapped`` is also set to true, the value ' + 'of the attribute defaults to 128.' + ), + value={ + 'type': int, + 'min': 0, + 'max': 255, + }, + ), + base.ExtraSpecValidator( + name='powervm:availability_priority', + description=( + 'Availability priority. The attribute priority of the server if ' + 'there is a processor failure and there are not enough resources ' + 'for all servers. VIOS and i5 need to remain high priority ' + 'default of 191. The value of the attribute defaults to 128.' + ), + value={ + 'type': int, + 'min': 0, + 'max': 255, + }, + ), + base.ExtraSpecValidator( + name='powervm:uncapped', + description=( + 'LPAR can use unused processor cycles that are beyond or exceed ' + 'the wanted setting of the attribute. This attribute is ' + 'supported only when ``powervm:dedicated_proc`` is set to false. ' + 'When ``powervm:dedicated_proc`` is set to false, ' + '``powervm:uncapped`` defaults to true.' + ), + value={ + 'type': bool, + }, + ), + base.ExtraSpecValidator( + name='powervm:dedicated_sharing_mode', + description=( + 'Sharing mode for dedicated processors. The attribute is ' + 'supported only when ``powervm:dedicated_proc`` is set to true.' + ), + value={ + 'type': str, + 'enum': ( + 'share_idle_procs', + 'keep_idle_procs', + 'share_idle_procs_active', + 'share_idle_procs_always', + ) + }, + ), + base.ExtraSpecValidator( + name='powervm:processor_compatibility', + description=( + 'A processor compatibility mode is a value that is assigned to a ' + 'logical partition by the hypervisor that specifies the processor ' + 'environment in which the logical partition can successfully ' + 'operate.' + ), + value={ + 'type': str, + 'enum': ( + 'default', + 'POWER6', + 'POWER6+', + 'POWER6_Enhanced', + 'POWER6+_Enhanced', + 'POWER7', + 'POWER8' + ), + }, + ), + base.ExtraSpecValidator( + name='powervm:shared_proc_pool_name', + description=( + 'Specifies the shared processor pool to be targeted during ' + 'deployment of a virtual machine.' + ), + value={ + 'type': str, + 'description': 'String with upper limit of 14 characters', + }, + ), + base.ExtraSpecValidator( + name='powervm:srr_capability', + description=( + 'If the value of simplified remote restart capability is set to ' + 'true for the LPAR, you can remote restart the LPAR to supported ' + 'CEC or host when the source CEC or host is down. The attribute ' + 'defaults to false.' + ), + value={ + 'type': bool, + }, + ), +] + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/api/validation/extra_specs/quota.py b/nova/api/validation/extra_specs/quota.py new file mode 100644 index 000000000000..89f56c107173 --- /dev/null +++ b/nova/api/validation/extra_specs/quota.py @@ -0,0 +1,103 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for ``quota`` namespaced extra specs.""" + +from nova.api.validation.extra_specs import base + + +EXTRA_SPEC_VALIDATORS = [] + + +# CPU, memory, disk IO and VIF quotas (VMWare) +for resource in ('cpu', 'memory', 'disk_io', 'vif'): + for key, fmt in ( + ('limit', int), + ('reservation', int), + ('shares_level', str), + ('shares_share', int) + ): + EXTRA_SPEC_VALIDATORS.append( + base.ExtraSpecValidator( + name=f'quota:{resource}_{key}', + description=( + 'The {} for {}. Only supported by the VMWare virt ' + 'driver.'.format(' '.join(key.split('_')), resource) + ), + value={ + 'type': fmt, + }, + ) + ) + + +# CPU quotas (libvirt) +for key in ('shares', 'period', 'quota'): + EXTRA_SPEC_VALIDATORS.append( + base.ExtraSpecValidator( + name=f'quota:cpu_{key}', + description=( + f'The quota {key} for CPU. Only supported by the libvirt ' + f'virt driver.' + ), + value={ + 'type': int, + 'min': 0, + }, + ) + ) + + +# Disk quotas (libvirt, HyperV) +for stat in ('read', 'write', 'total'): + for metric in ('bytes', 'iops'): + EXTRA_SPEC_VALIDATORS.append( + base.ExtraSpecValidator( + name=f'quota:disk_{stat}_{metric}_sec', + # NOTE(stephenfin): HyperV supports disk_total_{metric}_sec + # too; update + description=( + f'The quota {stat} {metric} for disk. Only supported ' + f'by the libvirt virt driver.' + ), + value={ + 'type': int, + 'min': 0, + }, + ) + ) + + +# VIF quotas (libvirt) +# TODO(stephenfin): Determine whether this should be deprecated now that +# nova-network is dead +for stat in ('inbound', 'outbound'): + for metric in ('average', 'peak', 'burst'): + EXTRA_SPEC_VALIDATORS.append( + base.ExtraSpecValidator( + name=f'quota:vif_{stat}_{metric}', + description=( + f'The quota {stat} {metric} for VIF. Only supported ' + f'by the libvirt virt driver.' + ), + value={ + 'type': int, + 'min': 0, + }, + ) + ) + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/api/validation/extra_specs/resources.py b/nova/api/validation/extra_specs/resources.py new file mode 100644 index 000000000000..2444d39fc15a --- /dev/null +++ b/nova/api/validation/extra_specs/resources.py @@ -0,0 +1,65 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for ``resources`` namespaced extra specs.""" + +import os_resource_classes + +from nova.api.validation.extra_specs import base + + +EXTRA_SPEC_VALIDATORS = [] + +for resource_class in os_resource_classes.STANDARDS: + EXTRA_SPEC_VALIDATORS.append( + base.ExtraSpecValidator( + name=f'resources{{group}}:{resource_class}', + description=f'The amount of resource {resource_class} requested.', + value={ + 'type': int, + }, + parameters=[ + { + 'name': 'group', + 'pattern': r'(_[a-zA-z0-9_]*|\d+)?', + }, + ], + ) + ) + +EXTRA_SPEC_VALIDATORS.append( + base.ExtraSpecValidator( + name='resources{group}:CUSTOM_{resource}', + description=( + 'The amount of resource CUSTOM_{resource} requested.' + ), + value={ + 'type': int, + }, + parameters=[ + { + 'name': 'group', + 'pattern': r'(_[a-zA-z0-9_]*|\d+)?', + }, + { + 'name': 'resource', + 'pattern': r'.+', + }, + ], + ) +) + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/api/validation/extra_specs/traits.py b/nova/api/validation/extra_specs/traits.py new file mode 100644 index 000000000000..60ae165955f5 --- /dev/null +++ b/nova/api/validation/extra_specs/traits.py @@ -0,0 +1,51 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for ``traits`` namespaced extra specs.""" + +import os_traits + +from nova.api.validation.extra_specs import base + + +EXTRA_SPEC_VALIDATORS = [] + +for trait in os_traits.get_traits(): + EXTRA_SPEC_VALIDATORS.append( + base.ExtraSpecValidator( + name=f'trait{{group}}:{trait}', + description=f'Require or forbid trait {trait}.', + value={ + 'type': str, + 'enum': [ + 'required', + 'forbidden', + ], + }, + parameters=[ + { + 'name': 'group', + 'pattern': r'(_[a-zA-z0-9_]*|\d+)?', + }, + { + 'name': 'trait', + 'pattern': r'[a-zA-Z0-9_]+', + }, + ], + ) + ) + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/api/validation/extra_specs/validators.py b/nova/api/validation/extra_specs/validators.py new file mode 100644 index 000000000000..18b80b2b4d5d --- /dev/null +++ b/nova/api/validation/extra_specs/validators.py @@ -0,0 +1,79 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for all extra specs known by nova.""" + +import re +import typing as ty + +from oslo_log import log as logging +from stevedore import extension + +from nova.api.validation.extra_specs import base +from nova import exception + +LOG = logging.getLogger(__name__) + +VALIDATORS: ty.Dict[str, base.ExtraSpecValidator] = {} + + +def validate(name: str, value: str, mode: str): + """Validate a given extra spec. + + :param name: Extra spec name. + :param value: Extra spec value. + :param mode: Validation mode; one of: strict, permissive, disabled + :raises: exception.ValidationError if validation fails. + """ + if mode == 'disabled': + return + + # attempt a basic lookup for extra specs without embedded parameters + if name in VALIDATORS: + VALIDATORS[name].validate(name, value) + return + + # if that failed, fallback to a linear search through the registry + for validator in VALIDATORS.values(): + if re.fullmatch(validator.name_regex, name): + validator.validate(name, value) + return + + if mode == 'permissive': # unregistered extra spec, ignore + return + + raise exception.ValidationError( + f"Validation failed; extra spec '{name}' does not appear to be a " + f"valid extra spec." + ) + + +def load_validators(): + global VALIDATORS + + def _report_load_failure(mgr, ep, err): + LOG.warning(u'Failed to load %s: %s', ep.module_name, err) + + mgr = extension.ExtensionManager( + 'nova.api.extra_spec_validators', + on_load_failure_callback=_report_load_failure, + invoke_on_load=False, + ) + for ext in mgr: + # TODO(stephenfin): Make 'register' return a dict rather than a list? + for validator in ext.plugin.register(): + VALIDATORS[validator.name] = validator + + +load_validators() diff --git a/nova/api/validation/extra_specs/vmware.py b/nova/api/validation/extra_specs/vmware.py new file mode 100644 index 000000000000..96a03d294a70 --- /dev/null +++ b/nova/api/validation/extra_specs/vmware.py @@ -0,0 +1,48 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Validators for ``vmware`` namespaced extra specs.""" + +from nova.api.validation.extra_specs import base + + +EXTRA_SPEC_VALIDATORS = [ + base.ExtraSpecValidator( + name='vmware:hw_version', + description=( + 'Specify the hardware version used to create images. In an ' + 'environment with different host versions, you can use this ' + 'parameter to place instances on the correct hosts.' + ), + value={ + 'type': str, + }, + ), + base.ExtraSpecValidator( + name='vmware:storage_policy', + description=( + 'Specify the storage policy used for new instances.' + '\n' + 'If Storage Policy-Based Management (SPBM) is not enabled, this ' + 'parameter is ignored.' + ), + value={ + 'type': str, + }, + ), +] + + +def register(): + return EXTRA_SPEC_VALIDATORS diff --git a/nova/tests/unit/api/openstack/compute/test_flavors_extra_specs.py b/nova/tests/unit/api/openstack/compute/test_flavors_extra_specs.py index a85eb7f717c8..141c1c09ec8f 100644 --- a/nova/tests/unit/api/openstack/compute/test_flavors_extra_specs.py +++ b/nova/tests/unit/api/openstack/compute/test_flavors_extra_specs.py @@ -14,6 +14,8 @@ # under the License. import mock +import testtools +import unittest import webob from nova.api.openstack.compute import flavors_extraspecs \ @@ -264,6 +266,88 @@ class FlavorsExtraSpecsTestV21(test.TestCase): self.assertRaises(self.bad_request, self.controller.create, req, 1, body=body) + # TODO(stephenfin): Wire the microversion up + @unittest.expectedFailure + def test_create_invalid_specs_strict(self): + """Test behavior of strict validator.""" + invalid_specs = { + 'hw:numa_nodes': 'foo', + 'hw:cpu_policy': 'sharrred', + 'foo': 'bar', + 'hw:cpu_policyyyyyyy': 'shared', + } + for key, value in invalid_specs.items(): + body = {'extra_specs': {key: value}} + req = self._get_request( + '1/os-extra_specs', use_admin_context=True, version='2.82', + ) + with testtools.ExpectedException( + self.bad_request, 'Validation failed; .*' + ): + self.controller.create(req, 1, body=body) + + # TODO(stephenfin): Wire the microversion up + @unittest.expectedFailure + def test_create_invalid_specs_permissive(self): + """Test behavior of permissive validator.""" + invalid_specs = { + 'hw:numa_nodes': 'foo', + 'hw:cpu_policy': 'sharrred', + } + for key, value in invalid_specs.items(): + body = {'extra_specs': {key: value}} + req = self._get_request( + '1/os-extra_specs?validation=permissive', + use_admin_context=True, version='2.82', + ) + with testtools.ExpectedException( + self.bad_request, 'Validation failed; .*', + ): + self.controller.create(req, 1, body=body) + + valid_specs = { + 'foo': 'bar', + 'hw:cpu_policyyyyyyy': 'shared', + } + for key, value in valid_specs.items(): + body = {'extra_specs': {key: value}} + req = self._get_request( + '1/os-extra_specs?validation=permissive', + use_admin_context=True, version='2.82', + ) + self.controller.create(req, 1, body=body) + + def test_create_invalid_specs_disabled(self): + """Test behavior of permissive validator.""" + valid_specs = { + 'hw:numa_nodes': 'foo', + 'hw:cpu_policy': 'sharrred', + 'foo': 'bar', + 'hw:cpu_policyyyyyyy': 'shared', + } + for key, value in valid_specs.items(): + body = {'extra_specs': {key: value}} + req = self._get_request( + '1/os-extra_specs?validation=disabled', + use_admin_context=True, version='2.82', + ) + self.controller.create(req, 1, body=body) + + # TODO(stephenfin): Wire the microversion up + @unittest.expectedFailure + def test_create_invalid_validator_mode(self): + """Test behavior with an invalid validator mode.""" + body = {'extra_specs': {'hw:numa_nodes': '1'}} + req = self._get_request( + '1/os-extra_specs?validation=foo', + use_admin_context=True, version='2.82', + ) + with testtools.ExpectedException( + self.bad_request, + 'Invalid input for query parameters validation.*', + ): + self.controller.create(req, 1, body=body) + @mock.patch('nova.objects.flavor._flavor_extra_specs_add') def test_create_valid_specs(self, mock_flavor_extra_specs): valid_specs = { @@ -366,3 +450,105 @@ class FlavorsExtraSpecsTestV21(test.TestCase): use_admin_context=True) self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, req, 1, 'hw:numa_nodes', body=body) + + # TODO(stephenfin): Wire the microversion up + @unittest.expectedFailure + def test_update_invalid_specs_strict(self): + """Test behavior of strict validator.""" + invalid_specs = { + 'hw:numa_nodes': 'foo', + 'hw:cpu_policy': 'sharrred', + 'foo': 'bar', + 'hw:cpu_policyyyyyyy': 'shared', + } + for key, value in invalid_specs.items(): + body = {key: value} + req = self._get_request( + '1/os-extra_specs/{key}?validation=strict', + use_admin_context=True, version='2.82', + ) + with testtools.ExpectedException( + self.bad_request, 'Validation failed; .*' + ): + self.controller.update(req, 1, key, body=body) + + # TODO(stephenfin): Wire the microversion up + @unittest.expectedFailure + def test_update_invalid_specs_permissive(self): + """Test behavior of permissive validator.""" + invalid_specs = { + 'hw:numa_nodes': 'foo', + 'hw:cpu_policy': 'sharrred', + } + for key, value in invalid_specs.items(): + body = {key: value} + req = self._get_request( + f'1/os-extra_specs/{key}?validation=permissive', + use_admin_context=True, version='2.82', + ) + with testtools.ExpectedException( + self.bad_request, 'Validation failed; .*', + ): + self.controller.update(req, 1, key, body=body) + + valid_specs = { + 'foo': 'bar', + 'hw:cpu_policyyyyyyy': 'shared', + } + for key, value in valid_specs.items(): + body = {key: value} + req = self._get_request( + f'1/os-extra_specs/{key}?validation=permissive', + use_admin_context=True, version='2.82', + ) + self.controller.update(req, 1, key, body=body) + + def test_update_invalid_specs_disabled(self): + """Test behavior of permissive validator.""" + valid_specs = { + 'hw:numa_nodes': 'foo', + 'hw:cpu_policy': 'sharrred', + 'foo': 'bar', + 'hw:cpu_policyyyyyyy': 'shared', + } + for key, value in valid_specs.items(): + body = {key: value} + req = self._get_request( + f'1/os-extra_specs/{key}?validation=disabled', + use_admin_context=True, version='2.82', + ) + self.controller.update(req, 1, key, body=body) + + # TODO(stephenfin): Wire the microversion up + @unittest.expectedFailure + def test_update_invalid_validator_mode(self): + """Test behavior with an invalid validator mode.""" + key = 'hw:numa_nodes' + body = {'hw:numa_nodes': '1'} + req = self._get_request( + '1/os-extra_specs/{key}?validation=foo', + use_admin_context=True, version='2.82', + ) + with testtools.ExpectedException( + self.bad_request, + 'Invalid input for query parameters validation.*', + ): + self.controller.update(req, 1, key, body=body) + + @mock.patch('nova.objects.flavor._flavor_extra_specs_add') + def test_update_valid_specs(self, mock_flavor_extra_specs): + valid_specs = { + 'hide_hypervisor_id': 'true', + 'hw:numa_nodes': '1', + 'hw:numa_cpus.0': '0-3,8-9,11,10', + } + mock_flavor_extra_specs.side_effect = return_create_flavor_extra_specs + + for key, value in valid_specs.items(): + body = {key: value} + req = self._get_request( + f'1/os-extra_specs/{key}', use_admin_context=True, + version='2.82', + ) + res_dict = self.controller.update(req, 1, key, body=body) + self.assertEqual(value, res_dict[key]) diff --git a/nova/tests/unit/api/validation/__init__.py b/nova/tests/unit/api/validation/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/nova/tests/unit/api/validation/extra_specs/__init__.py b/nova/tests/unit/api/validation/extra_specs/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/nova/tests/unit/api/validation/extra_specs/test_validators.py b/nova/tests/unit/api/validation/extra_specs/test_validators.py new file mode 100644 index 000000000000..8aa51ff30955 --- /dev/null +++ b/nova/tests/unit/api/validation/extra_specs/test_validators.py @@ -0,0 +1,129 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ddt +import testtools + +from nova.api.validation.extra_specs import validators +from nova import exception +from nova import test + + +@ddt.ddt +class TestValidators(test.NoDBTestCase): + + @ddt.data('strict', 'permissive', 'disabled') + def test_spec(self, policy): + invalid_specs = ( + ('hw:cpu_realtime_maskk', '^0'), + ('hhw:cpu_realtime_mask', '^0'), + ('w:cpu_realtime_mask', '^0'), + ('hw:cpu_realtime_mas', '^0'), + ('hw_cpu_realtime_mask', '^0'), + ('foo', 'bar'), + ) + for key, value in invalid_specs: + if policy == 'strict': + with testtools.ExpectedException(exception.ValidationError): + validators.validate(key, value, policy) + else: + validators.validate(key, value, policy) + + @ddt.data('strict', 'permissive', 'disabled') + def test_value__str(self, policy): + valid_specs = ( + # patterns + ('hw:cpu_realtime_mask', '^0'), + ('hw:cpu_realtime_mask', '^0,2-3,1'), + ('hw:mem_page_size', 'large'), + ('hw:mem_page_size', '2kbit'), + ('hw:mem_page_size', '1GB'), + # enums + ('hw:cpu_thread_policy', 'prefer'), + ('hw:emulator_threads_policy', 'isolate'), + ('hw:pci_numa_affinity_policy', 'legacy'), + ) + for key, value in valid_specs: + validators.validate(key, value, policy) + + invalid_specs = ( + # patterns + ('hw:cpu_realtime_mask', '0'), + ('hw:cpu_realtime_mask', '^0,2-3,b'), + ('hw:mem_page_size', 'largest'), + ('hw:mem_page_size', '2kbits'), + ('hw:mem_page_size', '1gigabyte'), + # enums + ('hw:cpu_thread_policy', 'preferred'), + ('hw:emulator_threads_policy', 'iisolate'), + ('hw:pci_numa_affinity_policy', 'lgacy'), + ) + for key, value in invalid_specs: + if policy in ('strict', 'permissive'): + with testtools.ExpectedException(exception.ValidationError): + validators.validate(key, value, policy) + else: + validators.validate(key, value, policy) + + @ddt.data('strict', 'permissive', 'disabled') + def test_value__int(self, policy): + valid_specs = ( + ('hw:numa_nodes', '1'), + ('os:monitors', '1'), + ('powervm:shared_weight', '1'), + ('os:monitors', '8'), + ('powervm:shared_weight', '255'), + ) + for key, value in valid_specs: + validators.validate(key, value, 'strict') + + invalid_specs = ( + ('hw:serial_port_count', 'five'), # NaN + ('hw:serial_port_count', '!'), # NaN + ('hw:numa_nodes', '0'), # has min + ('os:monitors', '0'), # has min + ('powervm:shared_weight', '-1'), # has min + ('os:monitors', '9'), # has max + ('powervm:shared_weight', '256'), # has max + ) + for key, value in invalid_specs: + if policy in ('strict', 'permissive'): + with testtools.ExpectedException(exception.ValidationError): + validators.validate(key, value, policy) + else: + validators.validate(key, value, policy) + + @ddt.data('strict', 'permissive', 'disabled') + def test_value__bool(self, policy): + valid_specs = ( + ('hw:cpu_realtime', '1'), + ('hw:cpu_realtime', '0'), + ('hw:mem_encryption', 'true'), + ('hw:boot_menu', 'y'), + ) + for key, value in valid_specs: + validators.validate(key, value, 'strict') + + invalid_specs = ( + ('hw:cpu_realtime', '2'), + ('hw:cpu_realtime', '00'), + ('hw:mem_encryption', 'tru'), + ('hw:boot_menu', 'yah'), + ) + for key, value in invalid_specs: + if policy in ('strict', 'permissive'): + with testtools.ExpectedException(exception.ValidationError): + validators.validate(key, value, policy) + else: + validators.validate(key, value, policy) diff --git a/requirements.txt b/requirements.txt index ec11b519eb80..099f7cabd722 100644 --- a/requirements.txt +++ b/requirements.txt @@ -71,3 +71,4 @@ python-dateutil>=2.5.3 # BSD zVMCloudConnector>=1.3.0;sys_platform!='win32' # Apache 2.0 License futurist>=1.8.0 # Apache-2.0 openstacksdk>=0.35.0 # Apache-2.0 +dataclasses>=0.7;python_version=='3.6' # Apache 2.0 License diff --git a/setup.cfg b/setup.cfg index 5266413a9581..bc0996136760 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,6 +46,20 @@ oslo.policy.policies = # list_rules method into a separate entry point rather than using the # aggregate method. nova = nova.policies:list_rules +nova.api.extra_spec_validators = + aggregate_instance_extra_specs = nova.api.validation.extra_specs.aggregate_instance_extra_specs + capabilities = nova.api.validation.extra_specs.capabilities + hw = nova.api.validation.extra_specs.hw + hw_rng = nova.api.validation.extra_specs.hw_rng + hw_video = nova.api.validation.extra_specs.hw_video + null = nova.api.validation.extra_specs.null + os = nova.api.validation.extra_specs.os + pci_passthrough = nova.api.validation.extra_specs.pci_passthrough + powervm = nova.api.validation.extra_specs.powervm + quota = nova.api.validation.extra_specs.quota + resources = nova.api.validation.extra_specs.resources + traits = nova.api.validation.extra_specs.traits + vmware = nova.api.validation.extra_specs.vmware nova.compute.monitors.cpu = virt_driver = nova.compute.monitors.cpu.virt_driver:Monitor nova.scheduler.driver = diff --git a/tox.ini b/tox.ini index 5a7441348cb9..fcb083667a4a 100644 --- a/tox.ini +++ b/tox.ini @@ -238,7 +238,7 @@ commands = bandit -r nova -x tests -n 5 -ll # E731 temporarily skipped because of the number of # these that have to be fixed enable-extensions = H106,H203,H904 -ignore = E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,E251,H405,W504,E731 +ignore = E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,E251,H405,W504,E731,H238 exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,tools/xenserver*,releasenotes # To get a list of functions that are more complex than 25, set max-complexity # to 25 and run 'tox -epep8'.