Merge "api: Add schema validation framework"

ironic/api/controllers/v1/shard.py

@@ -17,6 +17,7 @@ from ironic import api
 from ironic.api.controllers.v1 import utils as api_utils
 from ironic.api.controllers.v1 import versions
 from ironic.api import method
+from ironic.api.schemas.v1 import shard as schema
 from ironic.api import validation
 from ironic.common.i18n import _
 from ironic.common import metrics_utils
@@ -36,6 +37,8 @@ class ShardController(pecan.rest.RestController):
         min_version=versions.MINOR_82_NODE_SHARD,
         message=_('The API version does not allow shards'),
     )
+    @validation.request_query_schema(schema.index_request_query)
+    @validation.response_body_schema(schema.index_response_body)
     def get_all(self):
         """Retrieve a list of shards.
 
@@ -53,6 +56,6 @@ class ShardController(pecan.rest.RestController):
         min_version=versions.MINOR_82_NODE_SHARD,
         message=_('The API version does not allow shards'),
     )
-    def get_one(self, __):
+    def get_one(self, _):
         """Explicitly do not support getting one."""
         pecan.abort(404)

ironic/api/schemas/v1/shard.py

@@ -0,0 +1,39 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# TODO(stephenfin): Switch additionalProperties to False in a future version
+index_request_query = {
+    'type': 'object',
+    'properties': {},
+    'required': [],
+    'additionalProperties': True,
+}
+
+index_response_body = {
+    'type': 'object',
+    'properties': {
+        'shards': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'count': {'type': 'integer', 'minimum': 1},
+                    'name': {'type': 'string'},
+                },
+                'required': ['count', 'name'],
+                'additionalProperties': False,
+            },
+        },
+    },
+    'required': ['shards'],
+    'additionalProperties': False,
+}

ironic/api/validation/__init__.py

@@ -10,12 +10,17 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+"""API request/response validating middleware."""
+
 import functools
+import inspect
 import typing as ty
 
+from oslo_serialization import jsonutils
 from webob import exc as webob_exc
 
 from ironic import api
+from ironic.api.validation import validators
 from ironic.common.i18n import _
 
 
@@ -70,3 +75,291 @@ def api_version(
         return wrapper
 
     return add_validator
+
+
+class Schemas:
+    """A microversion-aware schema container.
+
+    Allow definition and retrieval of schemas on a microversion-aware basis.
+    """
+
+    def __init__(self) -> None:
+        self._schemas: list[
+            tuple[dict[str, object], ty.Optional[int], ty.Optional[int]]
+        ] = []
+
+    def add_schema(
+        self,
+        schema: tuple[dict[str, object]],
+        min_version: ty.Optional[int],
+        max_version: ty.Optional[int],
+    ) -> None:
+        self._schemas.append((schema, min_version, max_version))
+
+    def __call__(self) -> ty.Optional[dict[str, object]]:
+        for schema, min_version, max_version in self._schemas:
+            if (
+                min_version and not api.request.version.minor >= min_version
+            ) or (
+                max_version and not api.request.version.minor <= max_version
+            ):
+                continue
+
+            return schema
+
+        return None
+
+
+def _schema_validator(
+    schema: ty.Dict[str, ty.Any],
+    target: ty.Dict[str, ty.Any],
+    min_version: ty.Optional[int],
+    max_version: ty.Optional[int],
+    is_body: bool = True,
+):
+    """A helper method to execute JSON Schema Validation.
+
+    This method checks the request version whether matches the specified
+    ``max_version`` and ``min_version``. If the version range matches the
+    request, we validate ``schema`` against ``target``. A failure will result
+    in ``ValidationError`` being raised.
+
+    :param schema: The JSON Schema schema used to validate the target.
+    :param target: The target to be validated by the schema.
+    :param min_version: An integer indicating the minimum API version
+        ``schema`` applies against.
+    :param max_version: An integer indicating the maximum API version
+        ``schema`` applies against.
+    :param args: Positional arguments which passed into original method.
+    :param kwargs: Keyword arguments which passed into original method.
+    :param is_body: Whether ``target`` is a HTTP request body or not.
+    :returns: None.
+    :raises: ``ValidationError`` if validation fails.
+    """
+    # Only validate against the schema if it lies within
+    # the version range specified. Note that if both min
+    # and max are not specified the validator will always
+    # be run.
+    if (
+        (min_version and api.request.version.minor < min_version)
+        or (max_version and api.request.version.minor > max_version)
+    ):
+        return
+
+    schema_validator = validators.SchemaValidator(schema, is_body=is_body)
+    schema_validator.validate(target)
+
+
+def _extract_parameters(function):
+    sig = inspect.signature(function)
+    params = []
+
+    for param in sig.parameters.values():
+        if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD:
+            if param.name == 'self':  # skip validating self
+                continue
+
+            params.append(param)
+    return params
+
+
+def request_parameter_schema(
+    schema: ty.Dict[str, ty.Any],
+    min_version: ty.Optional[int] = None,
+    max_version: ty.Optional[int] = None,
+):
+    """Decorator for registering a request parameter schema on API methods.
+
+    ``schema`` will be used for validating request parameters just before
+    the API method is executed.
+
+    :param schema: The JSON Schema schema used to validate the target.
+    :param min_version: An integer indicating the minimum API version
+        ``schema`` applies against.
+    :param max_version: An integer indicating the maximum API version
+        ``schema`` applies against.
+    """
+
+    def add_validator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            # we need to convert positional arguments to a dict mapping token
+            # name to value so that we have a reference to compare against
+            parameters = _extract_parameters(func)
+            if func.__name__ in ('patch', 'post'):
+                # if this a create or update method, we need to ignore the
+                # request body parameter
+                parameters = parameters[:-1]
+
+            parameters = {
+                p.name: args[i + 1] for i, p in enumerate(parameters)
+                if p.name != '_' and p.default is p.empty
+            }
+            _schema_validator(
+                schema,
+                parameters,
+                min_version,
+                max_version,
+                is_body=True,
+            )
+            return func(*args, **kwargs)
+
+        if not hasattr(wrapper, 'request_parameter_schemas'):
+            wrapper.request_parameter_schemas = Schemas()
+
+        wrapper.request_parameter_schemas .add_schema(
+            schema, min_version, max_version
+        )
+
+        return wrapper
+
+    return add_validator
+
+
+def request_query_schema(
+    schema: ty.Dict[str, ty.Any],
+    min_version: ty.Optional[int] = None,
+    max_version: ty.Optional[int] = None,
+):
+    """Decorator for registering a request query string schema on API methods.
+
+    ``schema`` will be used for validating request query strings just before
+    the API method is executed.
+
+    :param schema: The JSON Schema schema used to validate the target.
+    :param min_version: An integer indicating the minimum API version
+        ``schema`` applies against.
+    :param max_version: An integer indicating the maximum API version
+        ``schema`` applies against.
+    """
+
+    def add_validator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            _schema_validator(
+                schema,
+                kwargs,
+                min_version,
+                max_version,
+                is_body=True,
+            )
+            return func(*args, **kwargs)
+
+        if not hasattr(wrapper, 'request_query_schemas'):
+            wrapper.request_query_schemas = Schemas()
+
+        wrapper.request_query_schemas .add_schema(
+            schema, min_version, max_version
+        )
+
+        return wrapper
+
+    return add_validator
+
+
+def request_body_schema(
+    schema: ty.Dict[str, ty.Any],
+    min_version: ty.Optional[str] = None,
+    max_version: ty.Optional[str] = None,
+):
+    """Decorator for registering a request body schema on API methods.
+
+    ``schema`` will be used for validating the request body just before the API
+    method is executed.
+
+    :param schema: The JSON Schema schema used to validate the target.
+    :param min_version: A string indicating the minimum API version ``schema``
+        applies against.
+    :param max_version: A string indicating the maximum API version ``schema``
+        applies against.
+    """
+
+    def add_validator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            parameters = _extract_parameters(func)
+            if not parameters:
+                # TODO(stephenfin): this would be a better check if we
+                # distinguished between 'create' operations (which should have
+                # at least one parameter, the body) and "update" operations
+                # (which should have at least two, the IDs and the body)
+                raise RuntimeError(
+                    'The ironic.api.method.body decorator must be placed '
+                    'outside the validation helpers to ensure it runs first.'
+                )
+
+            _schema_validator(
+                schema,
+                # The body argument will always be the last one
+                kwargs[parameters[-1].name],
+                min_version,
+                max_version,
+                is_body=True,
+            )
+            return func(*args, **kwargs)
+
+        if not hasattr(wrapper, 'request_body_schemas'):
+            wrapper.request_body_schemas = Schemas()
+
+        wrapper.request_body_schemas .add_schema(
+            schema, min_version, max_version
+        )
+
+        return wrapper
+
+    return add_validator
+
+
+def response_body_schema(
+    schema: ty.Dict[str, ty.Any],
+    min_version: ty.Optional[str] = None,
+    max_version: ty.Optional[str] = None,
+):
+    """Decorator for registering a response body schema on API methods.
+
+    ``schema`` will be used for validating the response body just after the API
+    method is executed.
+
+    :param schema: The JSON Schema schema used to validate the target.
+    :param min_version: A string indicating the minimum API version ``schema``
+        applies against.
+    :param max_version: A string indicating the maximum API version ``schema``
+        applies against.
+    """
+
+    def add_validator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            response = func(*args, **kwargs)
+
+            # FIXME(stephenfin): How is ironic/pecan doing jsonification? The
+            # below will fail on e.g. date-time fields
+
+            # NOTE(stephenfin): If our response is an object, we need to
+            # serialize and deserialize to convert e.g. date-time to strings
+            _body = jsonutils.dumps(response)
+
+            if _body == b'':
+                body = None
+            else:
+                body = jsonutils.loads(_body)
+
+            _schema_validator(
+                schema,
+                body,
+                min_version,
+                max_version,
+                is_body=True,
+            )
+            return response
+
+        if not hasattr(wrapper, 'response_body_schemas'):
+            wrapper.response_body_schemas = Schemas()
+
+        wrapper.response_body_schemas .add_schema(
+            schema, min_version, max_version
+        )
+
+        return wrapper
+
+    return add_validator

ironic/api/validation/validators.py

@@ -0,0 +1,121 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Internal implementation of request/response validating middleware."""
+
+import jsonschema
+from jsonschema import exceptions as jsonschema_exc
+from oslo_utils import timeutils
+from oslo_utils import uuidutils
+
+from ironic.common import exception
+from ironic.common.i18n import _
+
+
+@jsonschema.FormatChecker.cls_checks('date-time')
+def _validate_datetime_format(instance: object) -> bool:
+    # format checks constrain to the relevant primitive type
+    # https://github.com/OAI/OpenAPI-Specification/issues/3148
+    if not isinstance(instance, str):
+        return True
+    try:
+        timeutils.parse_isotime(instance)
+    except ValueError:
+        return False
+    else:
+        return True
+
+
+@jsonschema.FormatChecker.cls_checks('uuid')
+def _validate_uuid_format(instance: object) -> bool:
+    # format checks constrain to the relevant primitive type
+    # https://github.com/OAI/OpenAPI-Specification/issues/3148
+    if not isinstance(instance, str):
+        return True
+
+    return uuidutils.is_uuid_like(instance)
+
+
+class FormatChecker(jsonschema.FormatChecker):
+    """A FormatChecker can output the message from cause exception
+
+    We need understandable validation errors messages for users. When a
+    custom checker has an exception, the FormatChecker will output a
+    readable message provided by the checker.
+    """
+
+    def check(self, param_value, format):
+        """Check whether the param_value conforms to the given format.
+
+        :param param_value: the param_value to check
+        :type: any primitive type (str, number, bool)
+        :param str format: the format that param_value should conform to
+        :raises: :exc:`FormatError` if param_value does not conform to format
+        """
+
+        if format not in self.checkers:
+            return
+
+        # For safety reasons custom checkers can be registered with
+        # allowed exception types. Anything else will fall into the
+        # default formatter.
+        func, raises = self.checkers[format]
+        result, cause = None, None
+
+        try:
+            result = func(param_value)
+        except raises as e:
+            cause = e
+        if not result:
+            msg = '%r is not a %r' % (param_value, format)
+            raise jsonschema_exc.FormatError(msg, cause=cause)
+
+
+class SchemaValidator:
+    """A validator class
+
+    This class is changed from Draft202012Validator to add format checkers for
+    data formats that are common in the Ironic API as well as add better error
+    messages.
+    """
+
+    validator = None
+    validator_org = jsonschema.Draft202012Validator
+
+    def __init__(
+        self, schema, is_body=True
+    ):
+        self.is_body = is_body
+        validator_cls = jsonschema.validators.extend(self.validator_org)
+        format_checker = FormatChecker()
+        try:
+            self.validator = validator_cls(
+                schema, format_checker=format_checker
+            )
+        except Exception:
+            raise
+
+    def validate(self, *args, **kwargs):
+        try:
+            self.validator.validate(*args, **kwargs)
+        except jsonschema.ValidationError as e:
+            error_msg = _('Schema error: %s') % e.message
+            # Sometimes the root message is too generic, try to find a possible
+            # root cause:
+            cause = None
+            current = e
+            while current.context:
+                current = jsonschema.exceptions.best_match(current.context)
+                cause = current.message
+            if cause is not None:
+                error_msg += _('. Possible root cause: %s') % cause
+            raise exception.InvalidParameterValue(error_msg)

ironic/tests/unit/api/validation/test_validators.py

@@ -0,0 +1,46 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ironic.api.validation import validators
+from ironic.common import exception
+from ironic.tests import base as test_base
+
+
+class TestSchemaValidator(test_base.TestCase):
+
+    def test_uuid_format(self):
+        schema = {'type': 'string', 'format': 'uuid'}
+        validator = validators.SchemaValidator(schema)
+
+        # passes
+        validator.validate('d1903ad5-c774-4bfe-8cf4-8e08d8dbb4d3')
+
+        # fails
+        self.assertRaises(
+            exception.InvalidParameterValue,
+            validator.validate,
+            'invalid uuid'
+        )
+
+    def test_datetime_format(self):
+        schema = {'type': 'string', 'format': 'date-time'}
+        validator = validators.SchemaValidator(schema)
+
+        # passes
+        validator.validate('2019-10-12T07:20:50.52Z')
+
+        # fails
+        self.assertRaises(
+            exception.InvalidParameterValue,
+            validator.validate,
+            'invalid date-time'
+        )

releasenotes/notes/add_schema_validation_framework-eaac62cfecb132b0.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds a schema validation framework to the API. This allows for the validation
+    of incoming requests and outgoing responses against a JSON schema, right at
+    the beginning and end of the request processing pipeline.