From 5a64d71923d3e11686d9b790f46e349f2144b9b0 Mon Sep 17 00:00:00 2001 From: Artem Goncharov Date: Mon, 4 Nov 2024 17:45:54 +0100 Subject: [PATCH] Add placement resource_class and trait schemas Add schemas for 2 placement resources Change-Id: If41963bc9adea3a45ace2fde8f0a30457100cafb --- codegenerator/openapi/base.py | 52 ++-- codegenerator/openapi/placement.py | 241 +++++++++++++++++- .../openapi/placement_schemas/__init__.py | 0 .../placement_schemas/resource_class.py | 101 ++++++++ .../openapi/placement_schemas/trait.py | 89 +++++++ 5 files changed, 445 insertions(+), 38 deletions(-) create mode 100644 codegenerator/openapi/placement_schemas/__init__.py create mode 100644 codegenerator/openapi/placement_schemas/resource_class.py create mode 100644 codegenerator/openapi/placement_schemas/trait.py diff --git a/codegenerator/openapi/base.py b/codegenerator/openapi/base.py index aa3787a..c1ab2db 100644 --- a/codegenerator/openapi/base.py +++ b/codegenerator/openapi/base.py @@ -209,12 +209,7 @@ class OpenStackServerSourceBase: # Pecan base app framework = "pecan" contr = controller - elif not controller and action and hasattr(action, "func"): - # Placement base app - framework = "placement" - controller = action - contr = action - action = None + else: raise RuntimeError(f"Unsupported controller {controller}") # logging.debug("Actions: %s, Versioned methods: %s", actions, versioned_methods) @@ -506,32 +501,6 @@ class OpenStackServerSourceBase: path=path, ) - elif framework == "placement": - if callable(controller.func): - func = controller.func - # Get the path/op spec only when we have - # something to fill in - path_spec = openapi_spec.paths.setdefault( - path, PathSchema(parameters=path_params) - ) - operation_spec = getattr(path_spec, method.lower()) - if not operation_spec.operationId: - operation_spec.operationId = operation_id - if operation_tags: - operation_spec.tags.extend(operation_tags) - operation_spec.tags = list(set(operation_spec.tags)) - - self.process_operation( - func, - openapi_spec, - operation_spec, - path_resource_names, - controller=controller, - operation_name=func.__name__, - method=method, - path=path, - ) - else: logging.warning(controller.__dict__.items()) logging.warning(contr.__dict__.items()) @@ -620,6 +589,8 @@ class OpenStackServerSourceBase: ): operation_spec.openstack["min-ver"] = ( start_version.get_string() + if hasattr(start_version, "get_string") + else str(start_version) ) if ( @@ -649,6 +620,8 @@ class OpenStackServerSourceBase: ): operation_spec.openstack["max-ver"] = ( end_version.get_string() + if hasattr(end_version, "get_string") + else str(end_version) ) action_name = getattr(func, "wsgi_action", None) @@ -669,6 +642,7 @@ class OpenStackServerSourceBase: end_version, action_name, ) + print(f"Bodies: {body_schemas} {mode}") if hasattr(func, "_wsme_definition"): fdef = getattr(func, "_wsme_definition") @@ -904,6 +878,7 @@ class OpenStackServerSourceBase: action_name, ): # Body is not expected, exit (unless we are in the "action") + print(f"mode={mode}") if body_schemas is None or (body_schemas == [] and mode != "action"): return mime_type: str | None = "application/json" @@ -1220,10 +1195,18 @@ class OpenStackServerSourceBase: closure_locals = closure.nonlocals min_ver = closure_locals.get("min_version", start_version) if min_ver and not isinstance(min_ver, str): - min_ver = min_ver.get_string() + min_ver = ( + min_ver.get_string() + if hasattr(min_ver, "get_string") + else str(min_ver) + ) max_ver = closure_locals.get("max_version", end_version) if max_ver and not isinstance(max_ver, str): - max_ver = max_ver.get_string() + max_ver = ( + max_ver.get_string() + if hasattr(max_ver, "get_string") + else str(max_ver) + ) if "errors" in closure_locals: expected_errors = closure_locals["errors"] @@ -1317,7 +1300,6 @@ class OpenStackServerSourceBase: sig = inspect.signature(v) vals = sig.parameters.get("validators", None) if vals: - print(vals) sig2 = inspect.signature(vals.default[0]) schema_param = sig2.parameters.get("schema", None) if schema_param: diff --git a/codegenerator/openapi/placement.py b/codegenerator/openapi/placement.py index 72f4783..e392219 100644 --- a/codegenerator/openapi/placement.py +++ b/codegenerator/openapi/placement.py @@ -10,27 +10,39 @@ # License for the specific language governing permissions and limitations # under the License. # +import logging +import inspect +import re from multiprocessing import Process from pathlib import Path from ruamel.yaml.scalarstring import LiteralScalarString from codegenerator.common.schema import SpecSchema +from codegenerator.common.schema import PathSchema +from codegenerator.common.schema import ParameterSchema +from codegenerator.openapi import base from codegenerator.openapi.base import OpenStackServerSourceBase from codegenerator.openapi.utils import merge_api_ref_doc +from codegenerator.openapi.placement_schemas import resource_class +from codegenerator.openapi.placement_schemas import trait class PlacementGenerator(OpenStackServerSourceBase): URL_TAG_MAP = {"/versions": "version"} + RESOURCE_MODULES = [resource_class, trait] + + VERSIONED_METHODS: dict = {} + def _api_ver_major(self, ver): - return ver.ver_major + return ver.major def _api_ver_minor(self, ver): - return ver.ver_minor + return ver.minor def _api_ver(self, ver): - return (ver.ver_major, ver.ver_minor) + return (ver.major, ver.minor) def _generate(self, target_dir, args): from oslo_config import cfg @@ -42,6 +54,7 @@ class PlacementGenerator(OpenStackServerSourceBase): self.api_version = microversion.max_version_string() self.min_api_version = microversion.min_version_string() + self.VERSIONED_METHODS = microversion.VERSIONED_METHODS config = cfg.ConfigOpts() conf_fixture = self.useFixture(config_fixture.Config(config)) @@ -105,3 +118,225 @@ class PlacementGenerator(OpenStackServerSourceBase): proc.join() if proc.exitcode != 0: raise RuntimeError("Error generating Placement OpenAPI schema") + + def _process_route( + self, route, openapi_spec, ver_prefix=None, framework=None + ): + # Placement exposes "action" as controller in route defaults, all others - "controller" + if not ("controller" in route.defaults or "action" in route.defaults): + return + if "action" in route.defaults and "_methods" in route.defaults: + # placement 405 handler + return + # Path can be "/servers/{id}", but can be + # "/volumes/:volume_id/types/:(id)" - process + # according to the routes lib logic + path = ver_prefix if ver_prefix else "" + operation_spec = None + for part in route.routelist: + if isinstance(part, dict): + path += "{" + part["name"] + "}" + else: + path += part + + if path == "": + # placement has "" path - see weird explanation in the placement source code + return + + method = ( + route.conditions.get("method", "GET")[0] + if route.conditions + else "GET" + ) + + controller = route.defaults.get("controller") + action = route.defaults.get("action") + logging.info( + "Path: %s; method: %s; operation: %s", path, method, action + ) + + versioned_methods = self.VERSIONED_METHODS + controller_actions = {} + if not controller and action and hasattr(action, "func"): + # Placement base app + framework = "placement" + controller = action + contr = action + action = None + + closurevars = inspect.getclosurevars(controller.func) + nonlocals = closurevars.nonlocals + qualified_name: str | None = nonlocals.get("qualified_name") + if qualified_name: + action = qualified_name + + else: + raise RuntimeError(f"Unsupported controller {controller}") + + # Get Path elements + path_elements: list[str] = list(filter(None, path.split("/"))) + if path_elements and base.VERSION_RE.match(path_elements[0]): + path_elements.pop(0) + operation_tags = self._get_tags_for_url(path) + + # Build path parameters (/foo/{foo_id}/bar/{id} => $foo_id, $foo_bar_id) + # Since for same path we are here multiple times check presence of + # parameter before adding new params + path_params: list[ParameterSchema] = [] + path_resource_names: list[str] = [ + x.replace("-", "_") + for x in filter(lambda x: not x.startswith("{"), path_elements) + ] + for path_element in path_elements: + if "{" in path_element: + param_name = path_element.strip("{}") + global_param_name = ( + "_".join(path_resource_names) + f"_{param_name}" + ) + + param_ref_name = self._get_param_ref( + openapi_spec, + global_param_name, + param_name, + param_location="path", + path=path, + ) + # Ensure reference to the param is in the path_params + if param_ref_name not in [k.ref for k in list(path_params)]: + path_params.append(ParameterSchema(ref=param_ref_name)) + # Cleanup path_resource_names + # if len(path_resource_names) > 0 and VERSION_RE.match(path_resource_names[0]): + # # We should not have version prefix in the path_resource_names + # path_resource_names.pop(0) + if len(path_resource_names) == 0: + path_resource_names.append("root") + elif path_elements[-1].startswith("{"): + rn = path_resource_names[-1] + if rn.endswith("ies"): + rn = rn.replace("ies", "y") + elif rn.endswith("sses"): + rn = rn[:-2] + elif rn.endswith("statuses"): + rn = rn[:-2] + else: + rn = rn.rstrip("s") + path_resource_names[-1] = rn + + # Set operationId + operation_id = re.sub( + r"^(/?v[0-9.]*/)", + "", + "/".join([x.strip("{}") for x in path_elements]) + + f":{method.lower()}", # noqa + ) + + if action in versioned_methods: + # Normal REST operation with version bounds + (start_version, end_version) = (None, None) + + for versioned_method in sorted( + versioned_methods[action], key=lambda v: v[0] + ): + start_version = versioned_method[0] + # end_version = versioned_method[1] + func = versioned_method[2] + + # Placement defaults to max version when unset. Since for us + # setting of max_mv means deprecation ignore that. + if str(end_version) == str(self.api_version): + end_version = None + + # Get the path/op spec only when we have + # something to fill in + path_spec = openapi_spec.paths.setdefault( + path, PathSchema(parameters=path_params) + ) + operation_spec = getattr(path_spec, method.lower()) + if not operation_spec.operationId: + operation_spec.operationId = operation_id + operation_spec.tags.extend(operation_tags) + operation_spec.tags = list(set(operation_spec.tags)) + + self.process_operation( + func, + openapi_spec, + operation_spec, + path_resource_names, + controller=controller, + method=method, + operation_name=action.split(".")[-1], + start_version=start_version, + end_version=end_version, + ) + # NOTE(gtema): this looks weird, but + # placement has different MV functions for single operation. + # Would it have single function with schema decorators it + # would be easy to respect MV information. In current form + # there is no sense in processing them individually - it is + # single method in the end. Just getting jsonschema is tricky. + break + else: + if callable(controller.func): + func = controller.func + + # Get the path/op spec only when we have + # something to fill in + path_spec = openapi_spec.paths.setdefault( + path, PathSchema(parameters=path_params) + ) + operation_spec = getattr(path_spec, method.lower()) + if not operation_spec.operationId: + operation_spec.operationId = operation_id + if operation_tags: + operation_spec.tags.extend(operation_tags) + operation_spec.tags = list(set(operation_spec.tags)) + + self.process_operation( + func, + openapi_spec, + operation_spec, + path_resource_names, + controller=controller, + operation_name=func.__name__, + method=method, + path=path, + ) + + return operation_spec + + def _post_process_operation_hook( + self, openapi_spec, operation_spec, path: str | None = None + ): + """Hook to allow service specific generator to modify details""" + for resource_mod in self.RESOURCE_MODULES: + hook = getattr(resource_mod, "_post_process_operation_hook", None) + if hook: + hook(openapi_spec, operation_spec, path=path) + + def _get_schema_ref( + self, + openapi_spec, + name, + description=None, + schema_def=None, + action_name=None, + ): + # Invoke modularized schema _get_schema_ref + for resource_mod in self.RESOURCE_MODULES: + hook = getattr(resource_mod, "_get_schema_ref", None) + if hook: + (ref, mime_type, matched) = hook( + openapi_spec, name, description, schema_def, action_name + ) + if matched: + return (ref, mime_type) + + # Default + (ref, mime_type) = super()._get_schema_ref( + openapi_spec, + name, + description, + schema_def=schema_def, + action_name=action_name, + ) + return (ref, mime_type) diff --git a/codegenerator/openapi/placement_schemas/__init__.py b/codegenerator/openapi/placement_schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/codegenerator/openapi/placement_schemas/resource_class.py b/codegenerator/openapi/placement_schemas/resource_class.py new file mode 100644 index 0000000..3ed357b --- /dev/null +++ b/codegenerator/openapi/placement_schemas/resource_class.py @@ -0,0 +1,101 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +import copy + +from typing import Any + +from codegenerator.common.schema import TypeSchema +from codegenerator.common.schema import ParameterSchema + +RESOURCE_CLASS_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of one resource class.", + }, + "links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "href": {"type": "string", "format": "uri"}, + "rel": {"type": "string"}, + }, + }, + "readOnly": True, + }, + }, + "additionalProperties": False, +} + +RESOURCE_CLASSES_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "resource_classes": { + "type": "array", + "items": RESOURCE_CLASS_SCHEMA, + "description": "A list of resource_class objects.", + } + }, + "additionalProperties": False, +} + +RESOURCE_CLASS_UPDATE_REQUEST_SCHEMA: dict[str, Any] = { + "oneOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of one resource class.", + } + }, + "x-openstack": {"min-ver": "1.7"}, + } + ], + "discriminator": "microversion", +} + + +def _get_schema_ref( + openapi_spec, name, description=None, schema_def=None, action_name=None +) -> tuple[str | None, str | None, bool]: + mime_type: str = "application/json" + ref: str + if name == "Resource_ClassesList_Resource_ClassesResponse": + openapi_spec.components.schemas.setdefault( + name, TypeSchema(**RESOURCE_CLASSES_SCHEMA) + ) + ref = f"#/components/schemas/{name}" + elif name in ["Resource_ClassUpdate_Resource_ClassRequest"]: + openapi_spec.components.schemas.setdefault( + name, TypeSchema(**RESOURCE_CLASS_UPDATE_REQUEST_SCHEMA) + ) + ref = f"#/components/schemas/{name}" + + elif name in [ + "Resource_ClassGet_Resource_ClassResponse", + "Resource_ClassesCreate_Resource_ClassRequest", + "Resource_ClassesCreate_Resource_ClassResponse", + "Resource_ClassUpdate_Resource_ClassResponse", + ]: + openapi_spec.components.schemas.setdefault( + name, TypeSchema(**RESOURCE_CLASS_SCHEMA) + ) + ref = f"#/components/schemas/{name}" + + else: + return (None, None, False) + + return (ref, mime_type, True) diff --git a/codegenerator/openapi/placement_schemas/trait.py b/codegenerator/openapi/placement_schemas/trait.py new file mode 100644 index 0000000..ead6592 --- /dev/null +++ b/codegenerator/openapi/placement_schemas/trait.py @@ -0,0 +1,89 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +import copy + +from typing import Any + +from codegenerator.common.schema import TypeSchema +from codegenerator.common.schema import ParameterSchema + +TRAITS_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "traits": { + "type": "array", + "items": {"type": "string", "minLength": 1, "maxLength": 255}, + "description": "A list of resource_class objects.", + } + }, + "required": ["traits"], + "additionalProperties": False, +} + +TRAIT_LIST_PARAMETERS: dict[str, Any] = { + "name": { + "in": "query", + "name": "name", + "description": "A string to filter traits. The following options are available: `startswith` operator filters the traits whose name begins with a specific prefix, e.g. name=startswith:CUSTOM, `in` operator filters the traits whose name is in the specified list, e.g. name=in:HW_CPU_X86_AVX,HW_CPU_X86_SSE,HW_CPU_X86_INVALID_FEATURE.", + "schema": {"type": "string"}, + }, + "associated": { + "in": "query", + "name": "associated", + "description": "If this parameter has a true value, the returned traits will be those that are associated with at least one resource provider. Available values for the parameter are true and false.", + "schema": {"type": ["string", "boolean"], "enum": ["true", "false"]}, + "x-openstack": {"is-flag": True}, + }, +} + + +def _post_process_operation_hook( + openapi_spec, operation_spec, path: str | None = None +): + """Hook to allow service specific generator to modify details""" + operationId = operation_spec.operationId + + if operationId == "traits:get": + for key, val in TRAIT_LIST_PARAMETERS.items(): + openapi_spec.components.parameters.setdefault( + key, ParameterSchema(**val) + ) + ref = f"#/components/parameters/{key}" + + if ref not in [x.ref for x in operation_spec.parameters]: + operation_spec.parameters.append(ParameterSchema(ref=ref)) + + +def _get_schema_ref( + openapi_spec, name, description=None, schema_def=None, action_name=None +) -> tuple[str | None, str | None, bool]: + mime_type: str = "application/json" + ref: str + if name == "TraitsList_TraitsResponse": + openapi_spec.components.schemas.setdefault( + name, TypeSchema(**TRAITS_SCHEMA) + ) + ref = f"#/components/schemas/{name}" + + elif name in [ + "TraitGet_TraitResponse", + "TraitPut_TraitRequest", + "TraitPut_TraitResponse", + ]: + # Traits have no bodies + return (None, None, True) + + else: + return (None, None, False) + + return (ref, mime_type, True)