openstack/codegenerator
Code Issues Proposed changes

Update identity.domain schemas

Browse Source 
- update domain schema
- fix domain.config schemas
- update generator templates to deal with dict of dicts in domain.config
  case

Change-Id: I51a828d8b0729d2543186f68bc1e42c83b81d1ff
This commit is contained in:
Artem Goncharov
2024-06-11 15:08:15 +02:00
parent 5b8b677deb
commit 907e8a8f95
10 changed files with 294 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
codegenerator/common/__init__.py
View File

@@ -152,13 +152,14 @@ def find_resource_schema(
"""
try:
if "type" not in schema:
# Response of server create is a server or reservation_id
# if "oneOf" in schema:
# kinds = {}
# for kind in schema["oneOf"]:
# kinds.update(kind)
# schema["type"] = kinds["type"]
if "allOf" in schema:
# Response of server create is a server or reservation_id,
# identity.domain.group[config] is oneOf also
if "oneOf" in schema:
kinds = {}
for kind in schema["oneOf"]:
kinds.update(kind)
schema["type"] = kinds["type"]
elif "allOf" in schema:
# {'allOf': [
# {'type': 'integer', 'minimum': 0},
# {'default': 0}]
@@ -501,6 +502,18 @@ def get_resource_names_from_url(path: str):
"delete_key",
]:
path_resource_names = ["qos_spec"]
if path == "/v3/domains/{domain_id}/config":
path_resource_names = ["domain", "config"]
elif path == "/v3/domains/{domain_id}/config/{group}":
path_resource_names = ["domain", "config", "group"]
elif path == "/v3/domains/{domain_id}/config/{group}/{option}":
path_resource_names = ["domain", "config", "group", "option"]
elif path == "/v3/domains/config/default":
path_resource_names = ["domain", "config"]
elif path == "/v3/domains/config/{group}/default":
path_resource_names = ["domain", "config", "group"]
elif path == "/v3/domains/config/{group}/{option}/default":
path_resource_names = ["domain", "config", "group", "option"]
if path == "/v2.0/ports/{port_id}/bindings/{id}/activate":
path_resource_names = ["port", "binding"]

6
codegenerator/common/rust.py
View File

@@ -103,12 +103,16 @@ class String(BasePrimitiveType):
class JsonValue(BasePrimitiveType):
type_hint: str = "Value"
imports: set[str] = set(["serde_json::Value"])
builder_macros: set[str] = set(["setter(into)"])
def get_sample(self):
return "json!({})"
@property
def imports(self):
imports: set[str] = set(["serde_json::Value"])
return imports
class Option(BaseCombinedType):
base_type: str = "Option"

35
codegenerator/metadata.py
View File

@@ -326,6 +326,32 @@ class MetadataGenerator(BaseGenerator):
operation_key = "create"
elif method == "patch":
operation_key = "update"
if (
args.service_type == "identity"
and resource_name
in [
"domain/config",
"domain/config/group",
"domain/config/group/option",
]
and path.endswith("/default")
and method == "get"
):
operation_key = "default"
if (
args.service_type == "identity"
and resource_name
in [
"domain/config",
"domain/config/group",
"domain/config/group/option",
]
and path.endswith("/default")
and method == "head"
):
# No need in HEAD defaults
continue
if operation_key in resource_model:
raise RuntimeError("Operation name conflict")
@@ -838,6 +864,15 @@ def post_process_identity_operation(
operation.targets["rust-cli"].response_key = "role_inferences"
operation.targets["rust-sdk"].response_key = "role_inferences"
if resource_name == "domain/config/group":
operation.targets["rust-sdk"].response_key = "config"
if "rust-cli" in operation.targets:
operation.targets["rust-cli"].response_key = "config"
elif resource_name == "domain/config/group/option":
operation.targets["rust-sdk"].response_key = "config"
if "rust-cli" in operation.targets:
operation.targets["rust-cli"].response_key = "config"
if "rust-cli" in operation.targets:
if "auth/catalog" == resource_name:
operation.targets["rust-cli"].cli_full_command = operation.targets[

4
codegenerator/openapi/base.py
View File

@@ -31,7 +31,7 @@ from ruamel.yaml import YAML
from wsme import types as wtypes
VERSION_RE = re.compile(r"[Vv][0-9.]*")
VERSION_RE = re.compile(r"[Vv][0-9\.]*")
def get_referred_type_data(func, name: str):
@@ -1129,7 +1129,7 @@ class OpenStackServerSourceBase:
def _get_tags_for_url(self, url):
"""Return Tag (group) name based on the URL"""
# Drop version prefix
url = re.sub(r"^(/v[0-9.]*/)", "/", url)
url = re.sub(r"^(/v[0-9\.]*/)", "/", url)
for k, v in self.URL_TAG_MAP.items():
if url.startswith(k):

48
codegenerator/openapi/keystone.py
View File

@@ -39,7 +39,10 @@ from codegenerator.openapi.utils import merge_api_ref_doc
class KeystoneGenerator(OpenStackServerSourceBase):
URL_TAG_MAP = {
"/versions": "version",
"/domains/config": "domain-configuration",
"/domains/{domain_id}/config": "domain-configuration",
"/domains/{domain_id}/groups/{group_id}/roles": "roles",
"/domains/{domain_id}/users/{user_id}/roles": "roles",
}
RESOURCE_MODULES = [
@@ -178,10 +181,9 @@ class KeystoneGenerator(OpenStackServerSourceBase):
if hasattr(view, "view_class"):
controller = view.view_class
path = ""
path_elements = []
path: str = ""
path_elements: list[str] = []
operation_spec = None
tag_name = None
for part in route.rule.split("/"):
if not part:
@@ -190,18 +192,9 @@ class KeystoneGenerator(OpenStackServerSourceBase):
param = part.strip("<>").split(":")
path_elements.append("{" + param[-1] + "}")
else:
if not tag_name and part != "" and part != "v3":
tag_name = part
path_elements.append(part)
if not tag_name:
tag_name = "versions"
path = "/" + "/".join(path_elements)
if tag_name not in [x["name"] for x in openapi_spec.tags]:
openapi_spec.tags.append(
{"name": tag_name, "description": LiteralScalarString("")}
)
# Get rid of /v3 for further processing
path_elements = path_elements[1:]
@@ -210,14 +203,13 @@ class KeystoneGenerator(OpenStackServerSourceBase):
# parameter before adding new params
path_params: list[ParameterSchema] = []
path_resource_names: list[str] = []
operation_tags = self._get_tags_for_url(path)
for path_element in path_elements:
if "{" in path_element:
param_name = path_element.strip("{}")
global_param_name = (
"_".join(path_resource_names) + f"_{param_name}"
)
# if global_param_name == "_project_id":
# global_param_name = "project_id"
param_ref_name = f"#/components/parameters/{global_param_name}"
# Ensure reference to the param is in the path_params
if param_ref_name not in [
@@ -249,12 +241,17 @@ class KeystoneGenerator(OpenStackServerSourceBase):
else:
rn = rn.rstrip("s")
path_resource_names[-1] = rn
if path == "/v3/domains/{domain_id}/config/{group}":
path_resource_names.append("group")
elif path == "/v3/domains/config/{group}/{option}/default":
path_resource_names.append("group")
elif path == "/v3/domains/{domain_id}/config/{group}/{option}":
path_resource_names.extend(["group", "option"])
# Hack resource element names for domain configs
if path in [
"/v3/domains/config/{group}/default",
"/v3/domains/{domain_id}/config/{group}",
]:
path_resource_names = ["domains", "config", "group"]
elif path in [
"/v3/domains/config/{group}/{option}/default",
"/v3/domains/{domain_id}/config/{group}/{option}",
]:
path_resource_names = ["domains", "config", "group", "option"]
path_spec = openapi_spec.paths.setdefault(
path, PathSchema(parameters=path_params)
@@ -332,8 +329,8 @@ class KeystoneGenerator(OpenStackServerSourceBase):
operation_spec.description = LiteralScalarString(
doc or f"{method} operation on {path}"
)
if tag_name and tag_name not in operation_spec.tags:
operation_spec.tags.append(tag_name)
operation_spec.tags.extend(operation_tags)
operation_spec.tags = list(set(operation_spec.tags))
self.process_operation(
func,
@@ -451,6 +448,11 @@ class KeystoneGenerator(OpenStackServerSourceBase):
{"$ref": "#/components/headers/X-Subject-Token"},
)
# Ensure operation tags are existing
for tag in operation_spec.tags:
if tag not in [x["name"] for x in openapi_spec.tags]:
openapi_spec.tags.append({"name": tag, "description": None})
self._post_process_operation_hook(
openapi_spec, operation_spec, path=path
)

165
codegenerator/openapi/keystone_schemas/domain.py
View File

@@ -13,18 +13,49 @@
from typing import Any
from keystone.resource import schema as ks_schema
from codegenerator.common.schema import ParameterSchema
from codegenerator.common.schema import TypeSchema
DOMAIN_SCHEMA: dict[str, Any] = {
"type": "object",
"description": "A domain object",
"properties": {
"id": {"type": "string", "format": "uuid", "readOnly": True},
**ks_schema._domain_properties,
"name": {
"type": "string",
"description": "The name of the domain.",
"minLength": 1,
"maxLength": 255,
"pattern": r"[\S]+",
},
"description": {
"type": "string",
"description": "The description of the domain.",
},
"enabled": {
"type": "boolean",
"description": "If set to true, domain is enabled. If set to false, domain is disabled.",
},
"tags": {
"type": "array",
"items": {
"type": "string",
"pattern": "^[^,/]*$",
"minLength": 1,
"maxLength": 255,
},
},
"options": {
"type": "object",
"description": "The resource options for the domain. Available resource options are immutable.",
},
},
"additionalProperties": True,
}
DOMAIN_CONTAINER_SCHEMA: dict[str, Any] = {
"type": "object",
"properties": {"domain": DOMAIN_SCHEMA},
}
DOMAINS_SCHEMA: dict[str, Any] = {
@@ -32,55 +63,59 @@ DOMAINS_SCHEMA: dict[str, Any] = {
"properties": {"domains": {"type": "array", "items": DOMAIN_SCHEMA}},
}
DOMAIN_CONFIG_GROUP_LDAP = {
"type": "object",
"description": "An ldap object. Required to set the LDAP group configuration options.",
"properties": {
"url": {
"type": "string",
"format": "uri",
"description": "The LDAP URL.",
},
"user_tree_dn": {
"type": "string",
"description": "The base distinguished name (DN) of LDAP, from where all users can be reached. For example, ou=Users,dc=root,dc=org.",
},
},
"additionalProperties": True,
}
DOMAIN_CONFIG_GROUP_IDENTITY = {
"type": "object",
"description": "An identity object.",
"properties": {
"driver": {
"type": "string",
"description": "The Identity backend driver.",
},
},
"additionalProperties": True,
}
DOMAIN_CONFIGS_SCHEMA: dict[str, Any] = {
"type": "object",
"properties": {
"config": {
"type": "object",
"description": "A config object.",
"properties": {
"identity": DOMAIN_CONFIG_GROUP_IDENTITY,
"ldap": DOMAIN_CONFIG_GROUP_LDAP,
"additionalProperties": {
"type": "object",
"additionalProperties": True,
},
}
},
}
DOMAIN_CONFIG_SCHEMA: dict[str, Any] = {
"oneOf": [
DOMAIN_CONFIG_GROUP_IDENTITY,
DOMAIN_CONFIG_GROUP_LDAP,
]
DOMAIN_CONFIG_GROUP_SCHEMA: dict[str, Any] = {
"type": "object",
"properties": {
"config": {
"type": "object",
"description": "A config object.",
"additionalProperties": {
"type": "object",
"additionalProperties": True,
},
"maxProperties": 1,
}
},
}
DOMAIN_CONFIG_GROUP_OPTION_SCHEMA: dict[str, Any] = {
"type": "object",
"properties": {
"config": {
"type": "object",
"additionalProperties": True,
"maxProperties": 1,
}
},
}
DOMAIN_LIST_PARAMETERS: dict[str, dict] = {
"domain_name": {
"in": "query",
"name": "name",
"description": "Filters the response by a domain name.",
"schema": {"type": "string"},
},
"domain_enabled": {
"in": "query",
"name": "enabled",
"description": "If set to true, then only domains that are enabled will be returned, if set to false only that are disabled will be returned. Any value other than 0, including no value, will be interpreted as true.",
"schema": {"type": "boolean"},
},
}
@@ -88,7 +123,19 @@ def _post_process_operation_hook(
openapi_spec, operation_spec, path: str | None = None
):
"""Hook to allow service specific generator to modify details"""
pass
operationId = operation_spec.operationId
if operationId == "domains:get":
for (
key,
val,
) in DOMAIN_LIST_PARAMETERS.items():
openapi_spec.components.parameters.setdefault(
key, ParameterSchema(**val)
)
ref = f"#/components/parameters/{key}"
if ref not in [x.ref for x in operation_spec.parameters]:
operation_spec.parameters.append(ParameterSchema(ref=ref))
def _get_schema_ref(
@@ -102,24 +149,16 @@ def _get_schema_ref(
ref: str
# Domains
if name in [
"DomainsPostRequest",
"DomainsPostResponse",
"DomainGetResponse",
"DomainPatchRequest",
"DomainPatchResponse",
]:
openapi_spec.components.schemas.setdefault(
"Domain", TypeSchema(**DOMAIN_SCHEMA)
"Domain", TypeSchema(**DOMAIN_CONTAINER_SCHEMA)
)
ref = "#/components/schemas/Domain"
elif name == "DomainsPostRequest":
openapi_spec.components.schemas.setdefault(
name, TypeSchema(**ks_schema.domain_create)
)
ref = f"#/components/schemas/{name}"
elif name == "DomainPatchRequest":
openapi_spec.components.schemas.setdefault(
name, TypeSchema(**ks_schema.domain_update)
)
ref = f"#/components/schemas/{name}"
elif name == "DomainsGetResponse":
openapi_spec.components.schemas.setdefault(
name, TypeSchema(**DOMAINS_SCHEMA)
@@ -143,21 +182,31 @@ def _get_schema_ref(
)
ref = "#/components/schemas/DomainConfig"
elif name in [
"DomainsConfigDefaultGroupGetResponse",
"DomainsConfigGroupGetResponse",
"DomainsConfigGroupPatchRequest",
"DomainsConfigGroupPatchResponse",
"DomainsConfigGroupPatchResponse",
"DomainsConfigGroupPatchResponse",
"DomainsConfigDefaultGroupGetResponse",
]:
openapi_spec.components.schemas.setdefault(
"DomainConfigGroup",
TypeSchema(**DOMAIN_CONFIG_GROUP_SCHEMA),
)
ref = "#/components/schemas/DomainConfigGroup"
elif name in [
"DomainsConfigDefaultGroupOptionGetResponse",
"DomainsConfigGroupOptionPatchRequest",
"DomainsConfigGroupOptionPatchResponse",
"DomainsConfigGroupOptionGetResponse",
"DomainsConfigGroupOptionPatchRequest",
]:
openapi_spec.components.schemas.setdefault(
"DomainConfigGroup",
TypeSchema(**DOMAIN_CONFIG_SCHEMA),
"DomainConfigGroupOption",
TypeSchema(**DOMAIN_CONFIG_GROUP_OPTION_SCHEMA),
)
ref = "#/components/schemas/DomainConfigGroup"
ref = "#/components/schemas/DomainConfigGroupOption"
else:
return (None, None, False)

87
codegenerator/rust_cli.py
View File

@@ -97,11 +97,21 @@ class VecString(common.BasePrimitiveType):
class JsonValue(common_rust.JsonValue):
"""Arbitrary JSON value"""
imports: set[str] = set(["crate::common::parse_json", "serde_json::Value"])
clap_macros: set[str] = set(
['value_name="JSON"', "value_parser=parse_json"]
)
original_data_type: BaseCompoundType | BaseCompoundType | None = None
original_data_type: BaseCombinedType | BaseCompoundType | None = None
@property
def imports(self):
imports: set[str] = set(
["crate::common::parse_json", "serde_json::Value"]
)
if self.original_data_type and isinstance(
self.original_data_type, common_rust.Dictionary
):
imports.update(["std::collections::BTreeMap", "anyhow::Context"])
return imports
class StructInputField(common_rust.StructField):
@@ -602,6 +612,15 @@ class RequestTypeManager(common_rust.TypeManager):
original_data_type=original_data_type,
item_type=String(),
)
elif isinstance(type_model, model.Dictionary) and isinstance(
type_model.value_type, model.Dictionary
):
original_data_type = self.convert_model(type_model.value_type)
typ = JsonValue(
original_data_type=DictionaryInput(
value_type=original_data_type
)
)
if typ:
if model_ref:
@@ -638,8 +657,10 @@ class RequestTypeManager(common_rust.TypeManager):
field_data_type = field_data_type.item_type
elif isinstance(field_data_type, EnumGroupStruct):
field_data_type.is_required = field.is_required
elif isinstance(field_data_type, DictionaryInput) and isinstance(
field_data_type.value_type, common_rust.BaseCompoundType
elif isinstance(
field_data_type, DictionaryInput
) and not isinstance(
field_data_type.value_type, common_rust.BasePrimitiveType
):
dict_type_model = self._get_adt_by_reference(field.data_type)
simplified_data_type = JsonValue()
@@ -1176,29 +1197,49 @@ class RustCliGenerator(BaseGenerator):
isinstance(response_def.get("type"), list)
and "object" in response_def["type"]
):
(_, response_types) = openapi_parser.parse(
(root, response_types) = openapi_parser.parse(
response_def
)
response_type_manager.set_models(response_types)
if isinstance(root, model.Dictionary):
value_type = (
response_type_manager.convert_model(
root.value_type
)
)
# if not isinstance(value_type, common_rust.BasePrimitiveType):
# value_type = JsonValue(original_data_type=value_type)
root_dict = HashMapResponse(
value_type=value_type
)
response_type_manager.refs[
model.Reference(
name="Body", type=HashMapResponse
)
] = root_dict
if method == "patch" and not request_types:
# image patch is a jsonpatch based operation
# where there is no request. For it we need to
# look at the response and get writable
# parameters as a base
is_json_patch = True
if not args.find_implemented_by_sdk:
raise NotImplementedError
additional_imports.update(
[
"json_patch::{Patch, diff}",
"serde_json::json",
]
else:
response_type_manager.set_models(
response_types
)
(_, response_types) = openapi_parser.parse(
response_def, ignore_read_only=True
)
type_manager.set_models(response_types)
if method == "patch" and not request_types:
# image patch is a jsonpatch based operation
# where there is no request. For it we need to
# look at the response and get writable
# parameters as a base
is_json_patch = True
if not args.find_implemented_by_sdk:
raise NotImplementedError
additional_imports.update(
[
"json_patch::{Patch, diff}",
"serde_json::json",
]
)
(_, response_types) = openapi_parser.parse(
response_def, ignore_read_only=True
)
type_manager.set_models(response_types)
elif response_def["type"] == "string":
(root_dt, _) = openapi_parser.parse(response_def)

8
codegenerator/rust_sdk.py
View File

@@ -176,7 +176,13 @@ class BTreeMap(common_rust.Dictionary):
".map(|(k, v)| (k, v.map(Into::into)))"
)
else:
return "BTreeMap::<String, String>::new().into_iter()"
if isinstance(self.value_type, BTreeMap):
return (
f"BTreeMap::<String, BTreeMap<String, {self.value_type.value_type.type_hint}>>::new().into_iter()"
f".map(|(k, v)| (k, v.into_iter()))"
)
else:
return f"BTreeMap::<String, {self.value_type.type_hint}>::new().into_iter()"
def get_mandatory_init(self):
return ""

30
codegenerator/templates/rust_macros.j2
View File

@@ -70,12 +70,19 @@ Option
{%- macro sdk_builder_setter_btreemap(field) %}
{%- set is_opt = False if field.data_type.__class__.__name__ != "Option" else True %}
{%- set dt = field.data_type if not is_opt else field.data_type.item_type %}
{%- set val_type = field.data_type.value_type %}
{{ docstring(field.description, indent=4) }}
pub fn {{ field.local_name }}<I, K, V>(&mut self, iter: I) -> &mut Self
pub fn {{ field.local_name }}<I, K, V{{ ",K1, V1" if val_type.__class__.__name__ == "BTreeMap" else ""}}>(&mut self, iter: I) -> &mut Self
where
I: Iterator<Item = (K, V)>,
K: Into<Cow<'a, str>>,
V: Into<{{ dt.value_type.type_hint }}>,
{%- if val_type.__class__.__name__ != "BTreeMap" %}
V: Into<{{ dt.value_type.type_hint }}>,
{% else %}
V: Iterator<Item = (K1, V1)>,
K1: Into<Cow<'a, str>>,
V1: Into<{{ val_type.value_type.type_hint }}>,
{% endif%}
{
self.{{ field.local_name }}
{%- if field.is_optional %}
@@ -85,7 +92,17 @@ Option
.get_or_insert(None)
{%- endif %}
.get_or_insert_with(BTreeMap::new)
.extend(iter.map(|(k, v)| (k.into(), v.into())));
.extend(iter.map(|(k, v)| (
k.into(),
{%- if val_type.__class__.__name__ != "BTreeMap" %}
v.into()
{%- else %}
BTreeMap::from_iter(
v.into_iter()
.map(|(k1, v1)| {(k1.into(), v1.into())})
)
{%- endif %}
)));
self
}
{%- endmacro %}
@@ -149,6 +166,13 @@ Some({{ val }})
{{ dst_var }}.{{ param.remote_name }}(
serde_json::from_value::<{{ sdk_mod_path[-1] }}::{{ original_type.name }}>({{ val_var }}.to_owned())?
);
{%- elif original_type and original_type.__class__.__name__ == "DictionaryInput" and original_type.value_type and original_type.value_type.__class__.__name__ == "DictionaryInput" %}
{{ dst_var }}.{{ param.remote_name }}(
serde_json::from_value::<BTreeMap<String, BTreeMap<String, {{ original_type.value_type.value_type.type_hint }}>>>({{ val_var | replace("&", "") }}.clone())
.with_context(|| "Failed to parse `{{ param.local_name }}` as dict of dicts of {{ original_type.value_type.value_type.type_hint }}")?
.into_iter()
.map(|(k, v)| (k, v.into_iter())),
);
{%- else %}
{{ dst_var }}.{{ param.remote_name }}({{ val_var | replace("&", "" )}}.clone());
{%- endif %}

2
tools/generate_rust_identity.sh
View File

@@ -5,6 +5,7 @@ METADATA=metadata
DST=~/workspace/github/gtema/openstack
NET_RESOURCES=(
"auth"
"domain"
"group"
"os_federation"
"endpoint"
@@ -25,4 +26,5 @@ for resource in "${NET_RESOURCES[@]}"; do
cp -av "${WRK_DIR}/rust/openstack_sdk/src/api/identity/v3/${resource}" ${DST}/openstack_sdk/src/api/identity/v3
cp -av "${WRK_DIR}/rust/openstack_sdk/src/api/identity/v3/${resource}.rs" ${DST}/openstack_sdk/src/api/identity/v3
cp -av "${WRK_DIR}/rust/openstack_cli/src/identity/v3/${resource}" ${DST}/openstack_cli/src/identity/v3
cp -av "${WRK_DIR}/rust/openstack_cli/tests/identity/v3/${resource}" ${DST}/openstack_cli/tests/identity/v3
done;