Make openapi specs reproducible

Sort openapi spec to make it possible to put them under the version
control not taking care of changed ordering.

Change-Id: Ifb999f85a6d77e751cba8587c516c7d2c5c23644

codegenerator/common/rust.py

@@ -1107,7 +1107,11 @@ class TypeManager:
                 and (
                     (
                         name in unique_models
-                        and unique_models[name].hash_ != model_.reference.hash_
+                        and (
+                            unique_models[name].hash_ != model_.reference.hash_
+                            # maybe we previously decided to rename it
+                            or self.refs[unique_models[name]].name != name
+                        )
                     )
                     or name == self.root_name
                 )
@@ -1131,28 +1135,51 @@ class TypeManager:
                 if new_name not in unique_models:
                     # New name is still unused
                     model_data_type.name = new_name
+                    self.refs[model_.reference].name = new_name
                     unique_models[new_name] = model_.reference
                     # rename original model to the same naming scheme
                     other_model = unique_models.get(name)
-                    if other_model and other_model.parent:
+                    # Rename all other already processed models with name type and hash matching current model or the other model
-                        # Try adding parent_name as prefix
+                    for ref, some_model in self.refs.items():
+                        if (
+                            hasattr(some_model, "name")
+                            and some_model.name == name
+                        ):
+                            if (
+                                ref.type == model_.reference.type
+                                and ref.parent
+                                and ref.parent.name
+                                and (
+                                    (
+                                        other_model
+                                        and ref.hash_ == other_model.hash_
+                                    )
+                                    or ref.hash_ == model_.reference.hash_
+                                )
+                            ):
                                 new_other_name = (
                                     "".join(
                                         x.title()
-                                for x in other_model.parent.name.split("_")
+                                        for x in ref.parent.name.split("_")
                                     )
                                     + name
                                 )
+                                some_model.name = new_other_name
+                                unique_models[new_other_name] = some_model
                                 logging.debug(
-                            f"Renaming also {other_model} into {new_other_name} for consistency"
+                                    f"Renaming also {some_model} into {new_other_name} for consistency"
                                 )
-                        self.refs[other_model].name = new_other_name
+
-                        # other_model.name = new_other_name
-                        unique_models[new_other_name] = other_model
                 else:
                     if model_.reference.hash_ == unique_models[new_name].hash_:
-                        # not sure whether the new name should be save somewhere to be properly used in cli
+                        if name != self.refs[unique_models[name]].name:
-                        self.ignored_models.append(model_.reference)
+                            logging.debug(
+                                f"Found that same model {model_.reference} that we previously renamed to {self.refs[unique_models[name]].name}"
+                            )
+                            pass
+                        # not sure whether the new name should be save
+                        # somewhere to be properly used in cli
+                        # self.ignored_models.append(model_.reference)
                     elif isinstance(model_data_type, Struct):
                         # This is already an exceptional case (identity.mapping
                         # with remote being oneOf with multiple structs)
@@ -1199,8 +1226,9 @@ class TypeManager:
                 # image.metadef.namespace have weird occurences of itself
                 and model_.reference != unique_models[name]
             ):
-                # Ignore duplicated (or more precisely same) model
+                # We already have literally same model. Do nothing expecting
-                self.ignored_models.append(model_.reference)
+                # that filtering in the `get_subtypes` will do the rest.
+                pass
             elif name:
                 unique_models[name] = model_.reference
 
@@ -1209,12 +1237,18 @@ class TypeManager:
 
     def get_subtypes(self):
         """Get all subtypes excluding TLA"""
+        # Need to prevent literaly same objects to be emitted multiple times
+        # what may happen in case of deep nesting
+        emitted: set[str] = set()
         for k, v in self.refs.items():
             if (
                 k
                 and isinstance(v, (Enum, Struct, StringEnum))
                 and k.name != self.root_name
             ):
+                key = f"{k.type}:{getattr(v, 'name', '')}:{k.hash_}"
+                if key not in emitted:
+                    emitted.add(key)
                     yield v
             elif (
                 k
@@ -1222,6 +1256,9 @@ class TypeManager:
                 and isinstance(v, self.option_type_class)
             ):
                 if isinstance(v.item_type, Enum):
+                    key = f"{v.item_type}:{getattr(v, 'name', '')}:{k.hash_}"
+                    if key not in emitted:
+                        emitted.add(key)
                         yield v.item_type
 
     def get_root_data_type(self):

codegenerator/model.py

@@ -516,10 +516,10 @@ class JsonSchemaParser:
                         ) in [x.reference for x in results]:
                             raise NotImplementedError
                         else:
-                            obj.reference.name = new_name
                             logging.info(
                                 f"rename {obj.reference.name} to {new_name}"
                             )
+                            obj.reference.name = new_name
             results.append(obj)
 
         return obj

codegenerator/openapi/base.py

@@ -83,6 +83,56 @@ def get_referred_type_data(func, name: str):
         raise RuntimeError("Cannot get module the function was defined in")
 
 
+def sort_schema(data: Any):
+    def schema_tag_order(item):
+        orders = {
+            "openapi": 0,
+            "info": 1,
+            "jsonSchemaDialect": 2,
+            "servers": 3,
+            "paths": 4,
+            "components": 5,
+            "security": 6,
+            "webhooks": 7,
+            "tags": 8,
+            "externalDocs": 9,
+        }
+        return orders.get(item[0], item[0])
+
+    return {
+        key: sort_data(value)
+        for key, value in sorted(data.items(), key=schema_tag_order)
+    }
+
+
+def sort_data(data: Any):
+    def get_key(item: Any) -> str:
+        if isinstance(item, dict):
+            return str(item)
+        elif isinstance(item, (str, bool, int, float)):
+            return str(item)
+        elif item is None:
+            return ""
+        else:
+            raise RuntimeError(f"Cannot determine key for {item}")
+
+    if isinstance(data, dict):
+        return {
+            key: sort_data(value)
+            for key, value in sorted(data.items(), key=lambda item: item[0])
+        }
+    elif isinstance(data, list):
+        return [sort_data(item) for item in sorted(data, key=get_key)]
+    elif isinstance(data, tuple):
+        return [sort_data(item) for item in sorted(data, key=get_key)]
+    elif isinstance(data, (str, bool, int, float)):
+        return data
+    elif data is None:
+        return data
+    else:
+        raise RuntimeError(f"Cannot sort {data} [{type(data)}]")
+
+
 class OpenStackServerSourceBase:
     # A URL to Operation tag (OpenApi group) mapping. Can be used when first
     # non parameter path element grouping is not enough
@@ -120,6 +170,7 @@ class OpenStackServerSourceBase:
         with open(path) as fp:
             spec = yaml.load(fp)
 
+        if spec:
             return SpecSchema(**spec)
 
     def dump_openapi(self, spec, path, validate: bool, service_type: str):
@@ -131,8 +182,10 @@ class OpenStackServerSourceBase:
         yaml.indent(mapping=2, sequence=4, offset=2)
         with open(path, "w") as fp:
             yaml.dump(
+                sort_schema(
                     spec.model_dump(
                         exclude_none=True, exclude_defaults=True, by_alias=True
+                    )
                 ),
                 fp,
             )

codegenerator/tests/unit/test_openapi.py

@@ -0,0 +1,169 @@
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+from collections import OrderedDict
+import json
+from unittest import TestCase
+
+from codegenerator.openapi import base
+
+
+class TestSortSchema(TestCase):
+    def test_sort_dict(self):
+        self.assertEqual(
+            json.dumps({"a": "b", "b": 2, "c": "foo", "d": "bar"}),
+            json.dumps(
+                base.sort_data({"b": 2, "c": "foo", "d": "bar", "a": "b"})
+            ),
+        )
+
+    def test_sort_dict_of_dicts(self):
+        self.assertEqual(
+            json.dumps(
+                {
+                    "components": {
+                        "schemas": {
+                            "Bar": {"enum": ["1", "2", "4"], "type": "string"},
+                            "Foo": {
+                                "description": "foo",
+                                "properties": {
+                                    "m": {"type": ["null", "string"]},
+                                    "z": {"type": "string"},
+                                },
+                                "type": "object",
+                            },
+                        }
+                    }
+                }
+            ),
+            json.dumps(
+                base.sort_data(
+                    {
+                        "components": {
+                            "schemas": {
+                                "Foo": {
+                                    "type": "object",
+                                    "description": "foo",
+                                    "properties": {
+                                        "z": {"type": "string"},
+                                        "m": {"type": ["string", "null"]},
+                                    },
+                                },
+                                "Bar": {
+                                    "type": "string",
+                                    "enum": ["1", "4", "2"],
+                                },
+                            }
+                        }
+                    }
+                )
+            ),
+        )
+
+    def test_sort_spec(self):
+        self.assertEqual(
+            {
+                "openapi": "3.1.0",
+                "info": {"title": "foo"},
+                "components": {
+                    "schemas": {
+                        "Bar": {"type": "string", "enum": ["1", "2", "4"]},
+                        "Baz": {"enum": [1, 2.3, "4", True]},
+                        "Foo": {
+                            "description": "foo",
+                            "properties": {
+                                "m": {"type": ["null", "string"]},
+                                "z": {"type": "string"},
+                            },
+                            "type": "object",
+                        },
+                    }
+                },
+            },
+            base.sort_data(
+                {
+                    "components": {
+                        "schemas": {
+                            "Foo": {
+                                "type": "object",
+                                "description": "foo",
+                                "properties": {
+                                    "z": {"type": "string"},
+                                    "m": {"type": ["string", "null"]},
+                                },
+                            },
+                            "Baz": {"enum": [1, 2.3, "4", True]},
+                            "Bar": {"type": "string", "enum": ["1", "4", "2"]},
+                        }
+                    },
+                    "info": {"title": "foo"},
+                    "openapi": "3.1.0",
+                }
+            ),
+        )
+
+    def test_order(self):
+        expected = OrderedDict(
+            {
+                "projects_not-tags": OrderedDict(
+                    {
+                        "in": "query",
+                        "name": "not-tags",
+                        "schema": {
+                            "type": "string",
+                            "x-openstack": {
+                                "openapi": {
+                                    "schema": {
+                                        "explode": "false",
+                                        "items": {
+                                            "maxLength": 255,
+                                            "minLength": 1,
+                                            "pattern": "^[^,/]*$",
+                                            "type": "string",
+                                        },
+                                        "style": "form",
+                                        "type": "array",
+                                    }
+                                }
+                            },
+                        },
+                    }
+                )
+            }
+        )
+        unordered = {
+            "projects_not-tags": {
+                "name": "not-tags",
+                "in": "query",
+                "schema": {
+                    "type": "string",
+                    "x-openstack": {
+                        "openapi": {
+                            "schema": {
+                                "type": "array",
+                                "items": {
+                                    "type": "string",
+                                    "minLength": 1,
+                                    "maxLength": 255,
+                                    "pattern": "^[^,/]*$",
+                                },
+                                "style": "form",
+                                "explode": "false",
+                            }
+                        }
+                    },
+                },
+            }
+        }
+        self.assertEqual(
+            json.dumps(expected), json.dumps(base.sort_data(unordered))
+        )