Drop munch dependency

Importing munch inside of SDK is taking around 0.3 second. Itself it is
not a big problem, but it hurts on the openstackclient front. In
addition to that munch project does not seem to be actively maintained
and had no releases since 2 years.
Dropping this dependency at once is requiring quite a big rework so
instead copy a heavily stripped version of what we really require from
it. This helps us to gain performance improvement while giving time to
rework our code to come up with a decicion on how to deal with it.

Change-Id: I6612278ae798d48b296239e3359026584efb8a70
This commit is contained in:
Artem Goncharov 2022-11-18 17:42:19 +01:00
parent e84deaa56a
commit 6e5f34dba5
17 changed files with 271 additions and 78 deletions

View File

@ -436,7 +436,8 @@ class BaremetalCloudMixin:
"""List virtual ports attached to the bare metal machine.
:param string name_or_id: A machine name or UUID.
:returns: List of ``munch.Munch`` representing the ports.
:returns: List of ``openstack.Resource`` objects representing
the ports.
"""
machine = self.get_machine(name_or_id)
vif_ids = self.baremetal.list_node_vifs(machine)

View File

@ -13,7 +13,6 @@
# import types so that we can reference ListType in sphinx param declarations.
# We can't just use list, because sphinx gets confused by
# openstack.resource.Resource.list and openstack.resource2.Resource.list
import types # noqa
from openstack.cloud import _utils
from openstack.cloud import exc

View File

@ -1571,7 +1571,6 @@ class ComputeCloudMixin:
"""
return self.compute.aggregates(**filters)
# TODO(stephenfin): This shouldn't return a munch
def get_aggregate(self, name_or_id, filters=None):
"""Get an aggregate by name or ID.
@ -1590,10 +1589,8 @@ class ComputeCloudMixin:
:returns: An aggregate dict or None if no matching aggregate is
found.
"""
aggregate = self.compute.find_aggregate(
return self.compute.find_aggregate(
name_or_id, ignore_missing=True)
if aggregate:
return aggregate._to_munch()
def create_aggregate(self, name, availability_zone=None):
"""Create a new host aggregate.
@ -1804,11 +1801,10 @@ class ComputeCloudMixin:
item.pop('x_openstack_request_ids', None)
def _normalize_server(self, server):
import munch
ret = munch.Munch()
ret = utils.Munch()
# Copy incoming server because of shared dicts in unittests
# Wrap the copy in munch so that sub-dicts are properly munched
server = munch.Munch(server)
server = utils.Munch(server)
self._remove_novaclient_artifacts(server)
@ -1824,7 +1820,7 @@ class ComputeCloudMixin:
# from volume
image = server.pop('image', None)
if str(image) != image:
image = munch.Munch(id=image['id'])
image = utils.Munch(id=image['id'])
ret['image'] = image
# From original_names from sdk

View File

@ -108,8 +108,8 @@ class FloatingIPCloudMixin:
A string containing a jmespath expression for further filtering.
Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]"
:returns: A floating IP ``munch.Munch`` or None if no matching floating
IP is found.
:returns: A floating IP ``openstack.network.v2.floating_ip.FloatingIP``
or None if no matching floating IP is found.
"""
return _utils._get_entity(self, 'floating_ip', id, filters)
@ -168,7 +168,8 @@ class FloatingIPCloudMixin:
neutron. `get_external_ipv4_floating_networks` is what you should
almost certainly be using.
:returns: A list of floating IP pool ``munch.Munch``.
:returns: A list of floating IP pool
``openstack.network.v2.floating_ip.FloatingIP``.
"""
if not self._has_nova_extension('os-floating-ip-pools'):
@ -185,7 +186,8 @@ class FloatingIPCloudMixin:
"""List all available floating IPs.
:param filters: (optional) dict of filter conditions to push down
:returns: A list of floating IP ``munch.Munch``.
:returns: A list of floating IP
``openstack.network.v2.floating_ip.FloatingIP``.
"""
# If pushdown filters are specified and we do not have batched caching
@ -219,8 +221,7 @@ class FloatingIPCloudMixin:
:param id: ID of the floating ip.
:returns: A floating ip
`:class:`~openstack.network.v2.floating_ip.FloatingIP` or
``munch.Munch``.
`:class:`~openstack.network.v2.floating_ip.FloatingIP`.
"""
error_message = "Error getting floating ip with ID {id}".format(id=id)
@ -670,7 +671,7 @@ class FloatingIPCloudMixin:
:param nat_destination: The fixed network the server's port for the
FIP to attach to will come from.
:returns: The server ``munch.Munch``
:returns: The server ``openstack.compute.v2.server.Server``
:raises: OpenStackCloudException, on operation error.
"""
@ -842,7 +843,7 @@ class FloatingIPCloudMixin:
:param nat_destination: (optional) the name of the network of the
port to associate with the floating ip.
:returns: the updated server ``munch.Munch``
:returns: the updated server ``openstack.compute.v2.server.Server``
"""
if reuse:
f_ip = self.available_floating_ip(network=network)
@ -885,7 +886,7 @@ class FloatingIPCloudMixin:
the fixed IP to attach the
floating IP should be on
:returns: The updated server ``munch.Munch``
:returns: The updated server ``openstack.compute.v2.server.Server``
:raises: ``OpenStackCloudException``, on operation error.
"""
@ -1216,14 +1217,13 @@ class FloatingIPCloudMixin:
def _normalize_floating_ip(self, ip):
# Copy incoming floating ip because of shared dicts in unittests
# Only import munch when we really need it
import munch
location = self._get_current_location(
project_id=ip.get('owner'))
# This copy is to keep things from getting epically weird in tests
ip = ip.copy()
ret = munch.Munch(location=location)
ret = utils.Munch(location=location)
fixed_ip_address = ip.pop('fixed_ip_address', ip.pop('fixed_ip', None))
floating_ip_address = ip.pop('floating_ip_address', ip.pop('ip', None))
@ -1252,7 +1252,7 @@ class FloatingIPCloudMixin:
# In neutron's terms, Nova floating IPs are always ACTIVE
status = 'ACTIVE'
ret = munch.Munch(
ret = utils.Munch(
attached=attached,
fixed_ip_address=fixed_ip_address,
floating_ip_address=floating_ip_address,

View File

@ -201,7 +201,8 @@ class OrchestrationCloudMixin:
:param filters: a dict containing additional filters to use. e.g.
{'stack_status': 'CREATE_COMPLETE'}
:returns: a list of ``munch.Munch`` containing the stack description.
:returns: a list of ``openstack.orchestration.v1.stack.Stack``
containing the stack description.
:raises: ``OpenStackCloudException`` if something goes wrong during the
OpenStack API call.

View File

@ -21,6 +21,7 @@ from openstack.cloud import exc
from openstack import exceptions
from openstack.network.v2._proxy import Proxy
from openstack import proxy
from openstack import utils
class SecurityGroupCloudMixin:
@ -40,7 +41,8 @@ class SecurityGroupCloudMixin:
"""List all available security groups.
:param filters: (optional) dict of filter conditions to push down
:returns: A list of security group ``munch.Munch``.
:returns: A list of security group
``openstack.network.v2.security_group.SecurityGroup``.
"""
# Security groups not supported
@ -86,8 +88,9 @@ class SecurityGroupCloudMixin:
A string containing a jmespath expression for further filtering.
Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]"
:returns: A security group ``munch.Munch`` or None if no matching
security group is found.
:returns: A security group
``openstack.network.v2.security_group.SecurityGroup``
or None if no matching security group is found.
"""
return _utils._get_entity(
@ -97,7 +100,8 @@ class SecurityGroupCloudMixin:
""" Get a security group by ID
:param id: ID of the security group.
:returns: A security group ``munch.Munch``.
:returns: A security group
``openstack.network.v2.security_group.SecurityGroup``.
"""
if not self._has_secgroups():
raise exc.OpenStackCloudUnavailableFeature(
@ -126,7 +130,8 @@ class SecurityGroupCloudMixin:
on (admin-only).
:param string stateful: Whether the security group is stateful or not.
:returns: A ``munch.Munch`` representing the new security group.
:returns: A ``openstack.network.v2.security_group.SecurityGroup``
representing the new security group.
:raises: OpenStackCloudException on operation error.
:raises: OpenStackCloudUnavailableFeature if security groups are
@ -200,7 +205,8 @@ class SecurityGroupCloudMixin:
:param string name: New name for the security group.
:param string description: New description for the security group.
:returns: A ``munch.Munch`` describing the updated security group.
:returns: A ``openstack.network.v2.security_group.SecurityGroup``
describing the updated security group.
:raises: OpenStackCloudException on operation error.
"""
@ -288,7 +294,8 @@ class SecurityGroupCloudMixin:
on (admin-only).
:param string description:
Description of the rule, max 255 characters.
:returns: A ``munch.Munch`` representing the new security group rule.
:returns: A ``openstack.network.v2.security_group.SecurityGroup``
representing the new security group rule.
:raises: OpenStackCloudException on operation error.
"""
@ -439,9 +446,7 @@ class SecurityGroupCloudMixin:
# secgroups
def _normalize_secgroup(self, group):
import munch
ret = munch.Munch()
ret = utils.Munch()
# Copy incoming group because of shared dicts in unittests
group = group.copy()
@ -493,9 +498,7 @@ class SecurityGroupCloudMixin:
# secgroups
def _normalize_secgroup_rule(self, rule):
import munch
ret = munch.Munch()
ret = utils.Munch()
# Copy incoming rule because of shared dicts in unittests
rule = rule.copy()

View File

@ -15,8 +15,6 @@
import ipaddress
import socket
import munch
from openstack import _log
from openstack.cloud import exc
from openstack import utils
@ -551,7 +549,7 @@ def obj_to_munch(obj):
"""
if obj is None:
return None
elif isinstance(obj, munch.Munch) or hasattr(obj, 'mock_add_spec'):
elif isinstance(obj, utils.Munch) or hasattr(obj, 'mock_add_spec'):
# If we obj_to_munch twice, don't fail, just return the munch
# Also, don't try to modify Mock objects - that way lies madness
return obj
@ -563,14 +561,14 @@ def obj_to_munch(obj):
# the dict we get, but we also want it to fall through to object
# attribute processing so that we can also get the request_ids
# data into our resulting object.
instance = munch.Munch(obj)
instance = utils.Munch(obj)
else:
instance = munch.Munch()
instance = utils.Munch()
for key in dir(obj):
try:
value = getattr(obj, key)
# some attributes can be defined as a @propierty, so we can't assure
# some attributes can be defined as a @property, so we can't assure
# to have a valid value
# e.g. id in python-novaclient/tree/novaclient/v2/quotas.py
except AttributeError:

View File

@ -21,7 +21,6 @@ import warnings
import dogpile.cache
import keystoneauth1.exceptions
import keystoneauth1.session
import munch
import requests.models
import requestsexceptions
@ -551,11 +550,11 @@ class _OpenStackCloudMixin:
@property
def current_project(self):
"""Return a ``munch.Munch`` describing the current project"""
"""Return a ``utils.Munch`` describing the current project"""
return self._get_project_info()
def _get_project_info(self, project_id=None):
project_info = munch.Munch(
project_info = utils.Munch(
id=project_id,
name=None,
domain_id=None,
@ -581,11 +580,11 @@ class _OpenStackCloudMixin:
@property
def current_location(self):
"""Return a ``munch.Munch`` explaining the current cloud location."""
"""Return a ``utils.Munch`` explaining the current cloud location."""
return self._get_current_location()
def _get_current_location(self, project_id=None, zone=None):
return munch.Munch(
return utils.Munch(
cloud=self.name,
# TODO(efried): This is wrong, but it only seems to be used in a
# repr; can we get rid of it?
@ -596,11 +595,11 @@ class _OpenStackCloudMixin:
def _get_identity_location(self):
'''Identity resources do not exist inside of projects.'''
return munch.Munch(
return utils.Munch(
cloud=self.name,
region_name=None,
zone=None,
project=munch.Munch(
project=utils.Munch(
id=None,
name=None,
domain_id=None,

View File

@ -445,7 +445,7 @@ class Proxy(adapter.Adapter, Generic[T]):
be a subclass of :class:`~openstack.resource.Resource` with a
``from_id`` method.
:param value: The ID of a resource or an object of ``resource_type``
class if using an existing instance, or ``munch.Munch``,
class if using an existing instance, or ``utils.Munch``,
or None to create a new instance.
:param attrs: A dict containing arguments for forming the request
URL, if needed.

View File

@ -42,7 +42,6 @@ import warnings
import jsonpatch
from keystoneauth1 import adapter
from keystoneauth1 import discover
import munch
from requests import structures
from openstack import _log
@ -999,12 +998,12 @@ class Resource(dict):
@classmethod
def _from_munch(cls, obj, synchronized=True, connection=None):
"""Create an instance from a ``munch.Munch`` object.
"""Create an instance from a ``utils.Munch`` object.
This is intended as a temporary measure to convert between shade-style
Munch objects and original openstacksdk resources.
:param obj: a ``munch.Munch`` object to convert from.
:param obj: a ``utils.Munch`` object to convert from.
:param bool synchronized: whether this object already exists on server
Must be set to ``False`` for newly created objects.
"""
@ -1023,7 +1022,7 @@ class Resource(dict):
if isinstance(value, Resource):
return value.to_dict(_to_munch=to_munch)
elif isinstance(value, dict) and to_munch:
return munch.Munch(value)
return utils.Munch(value)
elif value and isinstance(value, list):
converted = []
for raw in value:
@ -1032,7 +1031,7 @@ class Resource(dict):
raw.to_dict(_to_munch=to_munch)
)
elif isinstance(raw, dict) and to_munch:
converted.append(munch.Munch(raw))
converted.append(utils.Munch(raw))
else:
converted.append(raw)
return converted
@ -1060,14 +1059,14 @@ class Resource(dict):
hasn't returned.
:param bool original_names: When True, use attribute names as they
were received from the server.
:param bool _to_munch: For internal use only. Converts to `munch.Munch`
:param bool _to_munch: For internal use only. Converts to `utils.Munch`
instead of dict.
:return: A dictionary of key/value pairs where keys are named
as they exist as attributes of this class.
"""
if _to_munch:
mapping = munch.Munch()
mapping = utils.Munch()
else:
mapping = {}
@ -1118,7 +1117,7 @@ class Resource(dict):
return mapping
# Compatibility with the munch.Munch.toDict method
# Compatibility with the utils.Munch.toDict method
toDict = to_dict
# Make the munch copy method use to_dict
copy = to_dict

View File

@ -20,10 +20,11 @@ import pprint
import sys
import fixtures
import munch
from oslotest import base
import testtools.content
from openstack import utils
_TRUE_VALUES = ('true', '1', 'yes')
@ -84,9 +85,9 @@ class TestCase(base.BaseTestCase):
def assertEqual(self, first, second, *args, **kwargs):
'''Munch aware wrapper'''
if isinstance(first, munch.Munch):
if isinstance(first, utils.Munch):
first = first.toDict()
if isinstance(second, munch.Munch):
if isinstance(second, utils.Munch):
second = second.toDict()
return super(TestCase, self).assertEqual(
first, second, *args, **kwargs)

View File

@ -30,6 +30,11 @@ from openstack.tests.unit import base
class TestCreateServer(base.TestCase):
def _compare_servers(self, exp, real):
self.assertDictEqual(
server.Server(**exp).to_dict(computed=False),
real.to_dict(computed=False),
)
def test_create_server_with_get_exception(self):
"""
@ -330,7 +335,7 @@ class TestCreateServer(base.TestCase):
json={'server': fake_server}),
])
self.assertEqual(
self.cloud._normalize_server(fake_create_server)['adminPass'],
admin_pass,
self.cloud.create_server(
name='server-name', image=dict(id='image-id'),
flavor=dict(id='flavor-id'),
@ -369,9 +374,9 @@ class TestCreateServer(base.TestCase):
])
# The wait returns non-password server
mock_wait.return_value = self.cloud._normalize_server(fake_server)
mock_wait.return_value = server.Server(**fake_server)
server = self.cloud.create_server(
new_server = self.cloud.create_server(
name='server-name', image=dict(id='image-id'),
flavor=dict(id='flavor-id'),
admin_pass=admin_pass, wait=True)
@ -381,8 +386,8 @@ class TestCreateServer(base.TestCase):
# Even with the wait, we should still get back a passworded server
self.assertEqual(
server['admin_password'],
self.cloud._normalize_server(fake_server_with_pass)['adminPass']
new_server['admin_password'],
fake_server_with_pass['adminPass']
)
self.assert_calls()

View File

@ -22,11 +22,10 @@ Tests Floating IP resource methods for Neutron
import copy
import datetime
import munch
from openstack.cloud import exc
from openstack.tests import fakes
from openstack.tests.unit import base
from openstack import utils
class TestFloatingIP(base.TestCase):
@ -570,7 +569,7 @@ class TestFloatingIP(base.TestCase):
}]})])
self.cloud.add_ips_to_server(
munch.Munch(
utils.Munch(
id='f80e3ad0-e13e-41d4-8e9c-be79bccdb8f7',
addresses={
"private": [{

View File

@ -14,13 +14,13 @@ import copy
import queue
from unittest import mock
import munch
from testscenarios import load_tests_apply_scenarios as load_tests # noqa
from openstack import exceptions
from openstack import proxy
from openstack import resource
from openstack.tests.unit import base
from openstack import utils
class DeleteableResource(resource.Resource):
@ -182,7 +182,7 @@ class TestProxyPrivate(base.TestCase):
res._update = mock.Mock()
cls._from_munch.return_value = res
m = munch.Munch(answer=42)
m = utils.Munch(answer=42)
attrs = {"first": "Brian", "last": "Curtin"}
result = self.fake_proxy._get_resource(cls, m, **attrs)

View File

@ -15,13 +15,13 @@ import json
from unittest import mock
from keystoneauth1 import adapter
import munch
import requests
from openstack import exceptions
from openstack import format
from openstack import resource
from openstack.tests.unit import base
from openstack import utils
class FakeResponse:
@ -947,14 +947,14 @@ class TestResource(base.TestCase):
self.assertEqual('bar', res.foo_alias)
self.assertTrue('foo' in res.keys())
self.assertTrue('foo_alias' in res.keys())
expected = munch.Munch({
expected = utils.Munch({
'id': None,
'name': 'test',
'location': None,
'foo': 'bar',
'foo_alias': 'bar'
})
actual = munch.Munch(res)
actual = utils.Munch(res)
self.assertEqual(expected, actual)
self.assertEqual(expected, res.toDict())
self.assertEqual(expected, res.to_dict())
@ -1035,7 +1035,7 @@ class TestResource(base.TestCase):
attr = resource.Body("body_attr")
value = "value"
orig = munch.Munch(body_attr=value)
orig = utils.Munch(body_attr=value)
sot = Test._from_munch(orig, synchronized=False)
self.assertIn("body_attr", sot._body.dirty)
@ -1046,7 +1046,7 @@ class TestResource(base.TestCase):
attr = resource.Body("body_attr")
value = "value"
orig = munch.Munch(body_attr=value)
orig = utils.Munch(body_attr=value)
sot = Test._from_munch(orig)
self.assertNotIn("body_attr", sot._body.dirty)

View File

@ -9,7 +9,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from collections.abc import Mapping
import hashlib
import queue
import string
@ -403,3 +403,196 @@ class TinyDAG:
def is_complete(self):
return len(self._done) == self.size()
# Importing Munch is a relatively expensive operation (0.3s) while we do not
# really even need much of it. Before we can rework all places where we rely on
# it we can have a reduced version.
class Munch(dict):
"""A slightly stripped version of munch.Munch class"""
def __init__(self, *args, **kwargs):
self.update(*args, **kwargs)
# only called if k not found in normal places
def __getattr__(self, k):
"""Gets key if it exists, otherwise throws AttributeError.
"""
try:
return object.__getattribute__(self, k)
except AttributeError:
try:
return self[k]
except KeyError:
raise AttributeError(k)
def __setattr__(self, k, v):
"""Sets attribute k if it exists, otherwise sets key k. A KeyError
raised by set-item (only likely if you subclass Munch) will
propagate as an AttributeError instead.
"""
try:
# Throws exception if not in prototype chain
object.__getattribute__(self, k)
except AttributeError:
try:
self[k] = v
except Exception:
raise AttributeError(k)
else:
object.__setattr__(self, k, v)
def __delattr__(self, k):
"""Deletes attribute k if it exists, otherwise deletes key k. A KeyError
raised by deleting the key--such as when the key is missing--will
propagate as an AttributeError instead.
"""
try:
# Throws exception if not in prototype chain
object.__getattribute__(self, k)
except AttributeError:
try:
del self[k]
except KeyError:
raise AttributeError(k)
else:
object.__delattr__(self, k)
def toDict(self):
"""Recursively converts a munch back into a dictionary.
"""
return unmunchify(self)
@property
def __dict__(self):
return self.toDict()
def __repr__(self):
"""Invertible* string-form of a Munch. """
return f'{self.__class__.__name__}({dict.__repr__(self)})'
def __dir__(self):
return list(self.keys())
def __getstate__(self):
"""Implement a serializable interface used for pickling.
See https://docs.python.org/3.6/library/pickle.html.
"""
return {k: v for k, v in self.items()}
def __setstate__(self, state):
"""Implement a serializable interface used for pickling.
See https://docs.python.org/3.6/library/pickle.html.
"""
self.clear()
self.update(state)
@classmethod
def fromDict(cls, d):
"""Recursively transforms a dictionary into a Munch via copy."""
return munchify(d, cls)
def copy(self):
return type(self).fromDict(self)
def update(self, *args, **kwargs):
"""
Override built-in method to call custom __setitem__ method that may
be defined in subclasses.
"""
for k, v in dict(*args, **kwargs).items():
self[k] = v
def get(self, k, d=None):
"""
D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.
"""
if k not in self:
return d
return self[k]
def setdefault(self, k, d=None):
"""
D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D
"""
if k not in self:
self[k] = d
return self[k]
def munchify(x, factory=Munch):
"""Recursively transforms a dictionary into a Munch via copy."""
# Munchify x, using `seen` to track object cycles
seen = dict()
def munchify_cycles(obj):
try:
return seen[id(obj)]
except KeyError:
pass
seen[id(obj)] = partial = pre_munchify(obj)
return post_munchify(partial, obj)
def pre_munchify(obj):
if isinstance(obj, Mapping):
return factory({})
elif isinstance(obj, list):
return type(obj)()
elif isinstance(obj, tuple):
type_factory = getattr(obj, "_make", type(obj))
return type_factory(munchify_cycles(item) for item in obj)
else:
return obj
def post_munchify(partial, obj):
if isinstance(obj, Mapping):
partial.update((k, munchify_cycles(obj[k])) for k in obj.keys())
elif isinstance(obj, list):
partial.extend(munchify_cycles(item) for item in obj)
elif isinstance(obj, tuple):
for (item_partial, item) in zip(partial, obj):
post_munchify(item_partial, item)
return partial
return munchify_cycles(x)
def unmunchify(x):
"""Recursively converts a Munch into a dictionary."""
# Munchify x, using `seen` to track object cycles
seen = dict()
def unmunchify_cycles(obj):
try:
return seen[id(obj)]
except KeyError:
pass
seen[id(obj)] = partial = pre_unmunchify(obj)
return post_unmunchify(partial, obj)
def pre_unmunchify(obj):
if isinstance(obj, Mapping):
return dict()
elif isinstance(obj, list):
return type(obj)()
elif isinstance(obj, tuple):
type_factory = getattr(obj, "_make", type(obj))
return type_factory(unmunchify_cycles(item) for item in obj)
else:
return obj
def post_unmunchify(partial, obj):
if isinstance(obj, Mapping):
partial.update((k, unmunchify_cycles(obj[k])) for k in obj.keys())
elif isinstance(obj, list):
partial.extend(unmunchify_cycles(v) for v in obj)
elif isinstance(obj, tuple):
for (value_partial, value) in zip(partial, obj):
post_unmunchify(value_partial, value)
return partial
return unmunchify_cycles(x)

View File

@ -9,7 +9,6 @@ jsonpatch!=1.20,>=1.16 # BSD
os-service-types>=1.7.0 # Apache-2.0
keystoneauth1>=3.18.0 # Apache-2.0
munch>=2.1.0 # MIT
decorator>=4.4.1 # BSD
jmespath>=0.9.0 # MIT
iso8601>=0.1.11 # MIT