Enable generation of manila openapi

- Add Manila stub
- Add support for newer decorators with response info
- add new jobs for building spec

Change-Id: I3aa578af4ca50297ad0f3860f69e33df59ac6f22

codegenerator/openapi/base.py

@@ -17,7 +17,7 @@ import importlib
 import inspect
 import logging
 from pathlib import Path
-from typing import Any
+from typing import Any, Callable
 import re
 
 from codegenerator.common.schema import ParameterSchema
@@ -365,25 +365,28 @@ class OpenStackServerSourceBase:
             for action, op_name in controller_actions.items():
                 logging.info("Action %s: %s", action, op_name)
                 (start_version, end_version) = (None, None)
+                action_impls: list[tuple[Callable, str | None, str | None]] = (
+                    []
+                )
                 if isinstance(op_name, str):
                     # wsgi action value is a string
                     if op_name in versioned_methods:
                         # ACTION with version bounds
-                        if len(versioned_methods[op_name]) > 1:
-                            raise RuntimeError(
-                                "Multiple versioned methods for action %s",
-                                action,
-                            )
                         for ver_method in versioned_methods[op_name]:
-                            start_version = ver_method.start_version
+                            action_impls.append(
-                            end_version = ver_method.end_version
+                                (
-                            func = ver_method.func
+                                    ver_method.func,
-                            logging.info("Versioned action %s", func)
+                                    ver_method.start_version,
-                        # operation_id += f"[{op_name}]"
+                                    ver_method.end_version,
+                                )
+                            )
+                            logging.info(
+                                "Versioned action %s", ver_method.func
+                            )
                     elif hasattr(contr, op_name):
                         # ACTION with no version bounds
                         func = getattr(contr, op_name)
-                        # operation_id += f"[{op_name}]"
+                        action_impls.append((func, None, None))
                         logging.info("Unversioned action %s", func)
                     else:
                         logging.error(
@@ -405,15 +408,20 @@ class OpenStackServerSourceBase:
                     if key and key in versioned_methods:
                         # ACTION with version bounds
                         if len(versioned_methods[key]) > 1:
-                            raise RuntimeError(
+                            logging.warn(
-                                "Multiple versioned methods for action %s",
+                                f"There are multiple callables for action {key} instead of multiple bodies"
-                                action,
                             )
                         for ver_method in versioned_methods[key]:
-                            start_version = ver_method.start_version
+                            action_impls.append(
-                            end_version = ver_method.end_version
+                                (
-                            func = ver_method.func
+                                    ver_method.func,
-                            logging.info("Versioned action %s", func)
+                                    ver_method.start_version,
+                                    ver_method.end_version,
+                                )
+                            )
+                            logging.info(
+                                "Versioned action %s", ver_method.func
+                            )
                     elif slf and key:
                         vm = getattr(slf, "versioned_methods", None)
                         if vm and key in vm:
@@ -424,12 +432,18 @@ class OpenStackServerSourceBase:
                                     action,
                                 )
                             for ver_method in vm[key]:
-                                start_version = ver_method.start_version
+                                action_impls.append(
-                                end_version = ver_method.end_version
+                                    (
-                                func = ver_method.func
+                                        ver_method.func,
-                            logging.info("Versioned action %s", func)
+                                        ver_method.start_version,
+                                        ver_method.end_version,
+                                    )
+                                )
+                            logging.info(
+                                "Versioned action %s", ver_method.func
+                            )
                     else:
-                        func = op_name
+                        action_impls.append((op_name, None, None))
 
                 # Get the path/op spec only when we have
                 # something to fill in
@@ -442,19 +456,20 @@ class OpenStackServerSourceBase:
                 operation_spec.tags.extend(operation_tags)
                 operation_spec.tags = list(set(operation_spec.tags))
 
-                self.process_operation(
+                for func, start_version, end_version in action_impls:
-                    func,
+                    self.process_operation(
-                    openapi_spec,
+                        func,
-                    operation_spec,
+                        openapi_spec,
-                    path_resource_names,
+                        operation_spec,
-                    controller=controller,
+                        path_resource_names,
-                    operation_name=action,
+                        controller=controller,
-                    method=method,
+                        operation_name=action,
-                    start_version=start_version,
+                        method=method,
-                    end_version=end_version,
+                        start_version=start_version,
-                    mode="action",
+                        end_version=end_version,
-                    path=path,
+                        mode="action",
-                )
+                        path=path,
+                    )
         elif framework == "pecan":
             if callable(controller):
                 func = controller
@@ -534,6 +549,22 @@ class OpenStackServerSourceBase:
             operation_name,
             func,
         )
+        # New decorators start having explicit null ApiVersion instead of being null
+        if (
+            start_version
+            and not isinstance(start_version, str)
+            and self._api_ver_major(start_version) in [0, None]
+            and self._api_ver_minor(start_version) in [0, None]
+        ):
+            start_version = None
+        if (
+            end_version
+            and not isinstance(end_version, str)
+            and self._api_ver_major(end_version) in [0, None]
+            and self._api_ver_minor(end_version) in [0, None]
+        ):
+            end_version = None
+
         deser_schema = None
         deser = getattr(controller, "deserializer", None)
         if deser:
@@ -583,8 +614,12 @@ class OpenStackServerSourceBase:
                     start_version.get_string()
                 )
 
-        if mode != "action" and end_version:
+        if (
-            if end_version.ver_major == 0:
+            mode != "action"
+            and end_version
+            and self._api_ver_major(end_version)
+        ):
+            if self._api_ver_major(end_version) == 0:
                 operation_spec.openstack.pop("max-ver", None)
                 operation_spec.deprecated = None
             else:
@@ -618,7 +653,11 @@ class OpenStackServerSourceBase:
             closure = inspect.getclosurevars(f)
             closure_locals = closure.nonlocals
             min_ver = closure_locals.get("min_version", start_version)
+            if min_ver and not isinstance(min_ver, str):
+                min_ver = min_ver.get_string()
             max_ver = closure_locals.get("max_version", end_version)
+            if max_ver and not isinstance(max_ver, str):
+                max_ver = max_ver.get_string()
 
             if "errors" in closure_locals:
                 expected_errors = closure_locals["errors"]
@@ -631,9 +670,14 @@ class OpenStackServerSourceBase:
                     ]
                 elif isinstance(expected_errors, int):
                     expected_errors = [str(expected_errors)]
-            if "request_body_schema" in closure_locals:
+            if "request_body_schema" in closure_locals or hasattr(
+                f, "_request_body_schema"
+            ):
                 # Body type is known through method decorator
-                obj = closure_locals["request_body_schema"]
+                obj = closure_locals.get(
+                    "request_body_schema",
+                    getattr(f, "_request_body_schema", {}),
+                )
                 if obj.get("type") in ["object", "array"]:
                     # We only allow object and array bodies
                     # To prevent type name collision keep module name part of the name
@@ -666,8 +710,22 @@ class OpenStackServerSourceBase:
 
                     ref_name = f"#/components/schemas/{typ_name}"
                     body_schemas.append(ref_name)
-            if "query_params_schema" in closure_locals:
+            if "response_body_schema" in closure_locals or hasattr(
-                obj = closure_locals["query_params_schema"]
+                f, "_response_body_schema"
+            ):
+                # Response type is known through method decorator - PERFECT
+                obj = closure_locals.get(
+                    "response_body_schema",
+                    getattr(f, "_response_body_schema", {}),
+                )
+                ser_schema = obj
+            if "query_params_schema" in closure_locals or hasattr(
+                f, "_request_query_schema"
+            ):
+                obj = closure_locals.get(
+                    "query_params_schema",
+                    getattr(f, "_request_query_schema", {}),
+                )
                 query_params_versions.append((obj, min_ver, max_ver))
 
             f = f.__wrapped__
@@ -704,7 +762,9 @@ class OpenStackServerSourceBase:
         if query_params_versions:
             so = sorted(
                 query_params_versions,
-                key=lambda d: d[1].split(".") if d[1] else (0, 0),
+                key=lambda d: (
+                    tuple(map(int, d[1].split("."))) if d[1] else (0, 0)
+                ),
             )
             for data, min_ver, max_ver in so:
                 self.process_query_parameters(

codegenerator/openapi/manila.py

@@ -0,0 +1,145 @@
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+from multiprocessing import Process
+from pathlib import Path
+
+from ruamel.yaml.scalarstring import LiteralScalarString
+
+from codegenerator.common.schema import (
+    SpecSchema,
+)
+from codegenerator.openapi.base import OpenStackServerSourceBase
+from codegenerator.openapi.utils import merge_api_ref_doc
+
+
+class ManilaGenerator(OpenStackServerSourceBase):
+    URL_TAG_MAP = {
+        "/versions": "version",
+    }
+
+    def _api_ver_major(self, ver):
+        return ver._ver_major
+
+    def _api_ver_minor(self, ver):
+        return ver._ver_minor
+
+    def _api_ver(self, ver):
+        return (ver._ver_major, ver._ver_minor)
+
+    def _generate(self, target_dir, args):
+        import fixtures
+        from oslo_config import cfg
+        from oslo_config import fixture as config_fixture
+        from oslo_concurrency import lockutils
+
+        from manila.api.openstack import api_version_request
+        from manila.api.v2 import router
+        from manila import rpc
+
+        # from placement import handler
+        from manila.common import config
+        from manila import coordination
+
+        self.api_version = api_version_request._MAX_API_VERSION
+        self.min_api_version = api_version_request._MIN_API_VERSION
+
+        CONF = config.CONF
+
+        lock_path = self.useFixture(fixtures.TempDir()).path
+        self.fixture = self.useFixture(config_fixture.Config(lockutils.CONF))
+        self.fixture.config(lock_path=lock_path, group="oslo_concurrency")
+        self.fixture.config(
+            disable_process_locking=True, group="oslo_concurrency"
+        )
+
+        rpc.init(CONF)
+
+        CONF.set_override(
+            "backend_url", "file://" + lock_path, group="coordination"
+        )
+        coordination.LOCK_COORDINATOR.start()
+
+        # config = cfg.ConfigOpts()
+        # conf_fixture = self.useFixture(config_fixture.Config(config))
+        # conf.register_opts(conf_fixture.conf)
+        # handler = handler.PlacementHandler(config=conf_fixture.conf)
+
+        self.router = router.APIRouter()
+
+        work_dir = Path(target_dir)
+        work_dir.mkdir(parents=True, exist_ok=True)
+
+        impl_path = Path(
+            work_dir,
+            "openapi_specs",
+            "shared_file_system",
+            f"v{self.api_version}.yaml",
+        )
+        impl_path.parent.mkdir(parents=True, exist_ok=True)
+
+        openapi_spec = self.load_openapi(impl_path)
+        if not openapi_spec:
+            openapi_spec = SpecSchema(
+                info=dict(
+                    title="OpenStack Shared-File-System API",
+                    description=LiteralScalarString(
+                        "Shared File System API provided by Manila service"
+                    ),
+                    version=self.api_version,
+                ),
+                openapi="3.1.0",
+                security=[{"ApiKeyAuth": []}],
+                components=dict(
+                    securitySchemes={
+                        "ApiKeyAuth": {
+                            "type": "apiKey",
+                            "in": "header",
+                            "name": "X-Auth-Token",
+                        }
+                    },
+                ),
+            )
+
+        for route in self.router.map.matchlist:
+            if route.routepath.endswith(".:(format)"):
+                continue
+            if route.routepath.startswith("/{project"):
+                continue
+            # if not route.routepath.startswith("/resource-lock"):
+            #     continue
+
+            self._process_route(route, openapi_spec)
+
+        self._sanitize_param_ver_info(openapi_spec, self.min_api_version)
+
+        if args.api_ref_src:
+            merge_api_ref_doc(
+                openapi_spec,
+                args.api_ref_src,
+                allow_strip_version=False,
+            )
+
+        self.dump_openapi(openapi_spec, impl_path, args.validate)
+
+        lnk = Path(impl_path.parent, "v2.yaml")
+        lnk.unlink(missing_ok=True)
+        lnk.symlink_to(impl_path.name)
+
+        return impl_path
+
+    def generate(self, target_dir, args):
+        proc = Process(target=self._generate, args=[target_dir, args])
+        proc.start()
+        proc.join()
+        if proc.exitcode != 0:
+            raise RuntimeError("Error generating Manila OpenAPI schema")

codegenerator/openapi_spec.py

@@ -63,6 +63,11 @@ class OpenApiSchemaGenerator(BaseGenerator):
 
         PlacementGenerator().generate(target_dir, args)
 
+    def generate_manila(self, target_dir, args):
+        from codegenerator.openapi.manila import ManilaGenerator
+
+        ManilaGenerator().generate(target_dir, args)
+
     def generate(
         self, res, target_dir, openapi_spec=None, operation_id=None, args=None
     ):
@@ -85,6 +90,8 @@ class OpenApiSchemaGenerator(BaseGenerator):
             self.generate_neutron(target_dir, args)
         elif args.service_type == "placement":
             self.generate_placement(target_dir, args)
+        elif args.service_type == "shared-file-system":
+            self.generate_manila(target_dir, args)
         else:
             raise RuntimeError(
                 "Service type %s is not supported", args.service_type

setup.cfg

@@ -43,6 +43,8 @@ network =
     neutron-vpnaas>=23.0
 placement =
     openstack-placement>=10.0
+shared-file-system =
+    manila>=18.0
 
 [mypy]
 show_column_numbers = true

tools/generate_openapi_specs.sh

@@ -30,3 +30,7 @@ if [ -z "$1" -o "$1" = "placement" ]; then
   openstack-codegenerator --work-dir wrk --target openapi-spec --service-type placement --api-ref-src ${API_REF_BUILD_ROOT}/placement/api-ref/build/html/index.html --validate
   sed -i "s/(?expanded=delete-resource-provider-inventories-detail#delete-resource-provider-inventories)//" wrk/openapi_specs/placement/v1.yaml
 fi
+
+if [ -z "$1" -o "$1" = "shared-file-system" ]; then
+  openstack-codegenerator --work-dir wrk --target openapi-spec --service-type shared-file-system --api-ref-src ${API_REF_BUILD_ROOT}/manila/api-ref/build/html/index.html --validate
+fi

zuul.d/openapi.yaml

@@ -239,6 +239,35 @@
         project: "opendev.org/openstack/placement"
         path: "/api-ref/build/html/index.html"
 
+- job:
+    name: codegenerator-openapi-shared-file-system-tips
+    parent: codegenerator-openapi-tips-base
+    description: |
+      Generate OpenAPI spec for Manila
+    required-projects:
+      - name: openstack/manila
+
+    vars:
+      openapi_service: shared-file-system
+      install_additional_projects:
+        - project: "opendev.org/openstack/manila"
+          name: "."
+
+- job:
+    name: codegenerator-openapi-shared-file-system-tips-with-api-ref
+    parent: codegenerator-openapi-shared-file-system-tips
+    description: |
+      Generate OpenAPI spec for Manila consuming API-REF
+    required-projects:
+      - name: openstack/manila
+
+    pre-run:
+      - playbooks/openapi/pre-api-ref.yaml
+    vars:
+      codegenerator_api_ref:
+        project: "opendev.org/openstack/manila"
+        path: "/api-ref/build/html/index.html"
+
 - job:
     name: codegenerator-tox-publish-openapi-specs
     parent: opendev-tox-docs
@@ -259,6 +288,8 @@
         soft: true
       - name: codegenerator-openapi-placement-tips-with-api-ref
         soft: true
+      - name: codegenerator-openapi-shared-file-system-tips-with-api-ref
+        soft: true
     pre-run:
       - playbooks/openapi/fetch.yaml
     vars:

zuul.d/project.yaml

@@ -13,6 +13,7 @@
         - codegenerator-openapi-load-balancing-tips-with-api-ref
         - codegenerator-openapi-network-tips-with-api-ref
         - codegenerator-openapi-placement-tips-with-api-ref
+        - codegenerator-openapi-shared-file-system-tips-with-api-ref
         - codegenerator-tox-publish-openapi-specs
     gate:
       jobs:
@@ -25,4 +26,5 @@
         - codegenerator-openapi-load-balancing-tips-with-api-ref
         - codegenerator-openapi-network-tips-with-api-ref
         - codegenerator-openapi-placement-tips-with-api-ref
+        - codegenerator-openapi-shared-file-system-tips-with-api-ref
         - codegenerator-tox-publish-openapi-specs