openstack
/
sushy
Code Issues Proposed changes

Lazily load message registries 

Postpone (potentially very large) Redfish message registries
download and processing up to the first access by the client.

The goal is to reduce the amount of unnecessary traffic and
CPU cycles.

Change-Id: Ief77d3ccce7fda7e6656c494060e47fe27b20f2e
Story: 2007442
Task: 39102
This commit is contained in:
Ilya Etingof 2020-03-17 19:44:30 +01:00
parent 5b07f04538
commit c737bf831c
3 changed files with 118 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
releasenotes/notes/lazily-load-registries-0e9441e435c2471d.yaml Normal file
View File

@ -0,0 +1,6 @@
---
fixes:
- |
Postpones (potentially very large) Redfish message registries download and
processing up to the first access by the client. The goal is to reduce
the amount of unnecessary traffic and CPU cycles.

80
sushy/main.py
View File

@ -12,6 +12,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import logging
import pkg_resources
import requests
@ -55,6 +56,49 @@ class ProtocolFeaturesSupportedField(base.CompositeField):
"""The select query parameter is supported"""
class LazyRegistries(collections.MutableMapping):
"""Download registries on demand.
Redfish message registries can be very large. On top of that,
they are not used frequently. Thus, let's not pull them off
the BMC unless the consumer is actually trying to use them.
:param service_root: Redfish service root object
:type service_root: Sushy
"""
def __init__(self, service_root):
self._service_root = service_root
self._registries = None
def __getitem__(self, key):
registries = self.registries
return registries[key]
def __setitem__(self, key, value):
registries = self.registries
registries[key] = value
def __delitem__(self, key):
registries = self.registries
del registries[key]
def __iter__(self):
registries = self.registries
return iter(registries or ())
def __len__(self):
registries = self.registries
return len(registries)
@property
def registries(self):
if self._registries is None:
self._registries = self._service_root.registries
return self._registries
class Sushy(base.ResourceBase):
identity = base.Field('Id', required=True)
@ -180,7 +224,7 @@ class Sushy(base.ResourceBase):
return system.SystemCollection(
self._conn, self._systems_path,
redfish_version=self.redfish_version,
registries=self.registries)
registries=self.lazy_registries)
def get_system(self, identity=None):
"""Given the identity return a System object
@ -203,7 +247,7 @@ class Sushy(base.ResourceBase):
return system.System(self._conn, identity,
redfish_version=self.redfish_version,
registries=self.registries)
registries=self.lazy_registries)
def get_chassis_collection(self):
"""Get the ChassisCollection object
@ -218,7 +262,7 @@ class Sushy(base.ResourceBase):
return chassis.ChassisCollection(self._conn, self._chassis_path,
redfish_version=self.redfish_version,
registries=self.registries)
registries=self.lazy_registries)
def get_chassis(self, identity=None):
"""Given the identity return a Chassis object
@ -241,7 +285,7 @@ class Sushy(base.ResourceBase):
return chassis.Chassis(self._conn, identity,
redfish_version=self.redfish_version,
registries=self.registries)
registries=self.lazy_registries)
def get_fabric_collection(self):
"""Get the FabricCollection object
@ -256,7 +300,7 @@ class Sushy(base.ResourceBase):
return fabric.FabricCollection(self._conn, self._fabrics_path,
redfish_version=self.redfish_version,
registries=self.registries)
registries=self.lazy_registries)
def get_fabric(self, identity):
"""Given the identity return a Fabric object
@ -266,7 +310,7 @@ class Sushy(base.ResourceBase):
"""
return fabric.Fabric(self._conn, identity,
redfish_version=self.redfish_version,
registries=self.registries)
registries=self.lazy_registries)
def get_manager_collection(self):
"""Get the ManagerCollection object
@ -281,7 +325,7 @@ class Sushy(base.ResourceBase):
return manager.ManagerCollection(self._conn, self._managers_path,
redfish_version=self.redfish_version,
registries=self.registries)
registries=self.lazy_registries)
def get_manager(self, identity=None):
"""Given the identity return a Manager object
@ -303,7 +347,7 @@ class Sushy(base.ResourceBase):
return manager.Manager(self._conn, identity,
redfish_version=self.redfish_version,
registries=self.registries)
registries=self.lazy_registries)
def get_session_service(self):
"""Get the SessionService object
@ -328,7 +372,8 @@ class Sushy(base.ResourceBase):
"""
return session.Session(
self._conn, identity,
redfish_version=self.redfish_version, registries=self.registries)
redfish_version=self.redfish_version,
registries=self.lazy_registries)
def get_update_service(self):
"""Get the UpdateService object
@ -342,7 +387,7 @@ class Sushy(base.ResourceBase):
return updateservice.UpdateService(
self._conn, self._update_service_path,
redfish_version=self.redfish_version,
registries=self.registries)
registries=self.lazy_registries)
def _get_registry_collection(self):
"""Get MessageRegistryFileCollection object
@ -374,7 +419,7 @@ class Sushy(base.ResourceBase):
return compositionservice.CompositionService(
self._conn, self._composition_service_path,
redfish_version=self.redfish_version,
registries=self.registries)
registries=self.lazy_registries)
def _get_standard_message_registry_collection(self):
"""Load packaged standard message registries
@ -421,3 +466,16 @@ class Sushy(base.ResourceBase):
self._public_connector) for r in provided})
return registries
@property
def lazy_registries(self):
"""Gets and combines all message registries together
Fetches all registries if any provided by Redfish service
and combines together with packaged standard registries.
:returns: dict of combined message registries where key is
Registry_name.Major_version.Minor_version and value is registry
itself.
"""
return LazyRegistries(self)

79
sushy/tests/unit/test_main.py
View File

@ -95,32 +95,32 @@ class MainTestCase(base.TestCase):
self.assertFalse(mock_Sushy_Connector.called)
@mock.patch.object(system, 'SystemCollection', autospec=True)
@mock.patch('sushy.Sushy.registries', autospec=True)
@mock.patch('sushy.Sushy.lazy_registries', autospec=True)
def test_get_system_collection(
self, mock_registries, mock_system_collection):
self, mock_lazy_registries, mock_system_collection):
self.root._standard_message_registries_path = None
self.root.get_system_collection()
mock_system_collection.assert_called_once_with(
self.root._conn, '/redfish/v1/Systems',
redfish_version=self.root.redfish_version,
registries=mock_registries
registries=mock_lazy_registries
)
@mock.patch.object(system, 'System', autospec=True)
@mock.patch('sushy.Sushy.registries', autospec=True)
def test_get_system(self, mock_registries, mock_system):
@mock.patch('sushy.Sushy.lazy_registries', autospec=True)
def test_get_system(self, mock_lazy_registries, mock_system):
self.root._standard_message_registries_path = None
self.root.get_system('fake-system-id')
mock_system.assert_called_once_with(
self.root._conn, 'fake-system-id',
redfish_version=self.root.redfish_version,
registries=mock_registries)
registries=mock_lazy_registries)
@mock.patch.object(system, 'SystemCollection', autospec=True)
@mock.patch.object(system, 'System', autospec=True)
@mock.patch('sushy.Sushy.registries', autospec=True)
@mock.patch('sushy.Sushy.lazy_registries', autospec=True)
def test_get_system_default_ok(
self, mock_registries, mock_system, mock_system_collection):
self, mock_lazy_registries, mock_system, mock_system_collection):
self.root._standard_message_registries_path = None
mock_system.path = 'fake-system-id'
mock_members = mock_system_collection.return_value.get_members
@ -129,18 +129,18 @@ class MainTestCase(base.TestCase):
mock_system_collection.assert_called_once_with(
self.root._conn, '/redfish/v1/Systems',
redfish_version=self.root.redfish_version,
registries=mock_registries
registries=mock_lazy_registries
)
mock_system.assert_called_once_with(
self.root._conn, 'fake-system-id',
redfish_version=self.root.redfish_version,
registries=mock_registries)
registries=mock_lazy_registries)
@mock.patch.object(system, 'SystemCollection', autospec=True)
@mock.patch.object(system, 'System', autospec=True)
@mock.patch('sushy.Sushy.registries', autospec=True)
@mock.patch('sushy.Sushy.lazy_registries', autospec=True)
def test_get_system_default_failure(
self, mock_registries, mock_system, mock_system_collection):
self, mock_lazy_registries, mock_system, mock_system_collection):
self.root._standard_message_registries_path = None
mock_members = mock_system_collection.return_value.get_members
mock_members.return_value = []
@ -148,7 +148,7 @@ class MainTestCase(base.TestCase):
mock_system_collection.assert_called_once_with(
self.root._conn, '/redfish/v1/Systems',
redfish_version=self.root.redfish_version,
registries=mock_registries
registries=mock_lazy_registries
)
@mock.patch.object(chassis, 'Chassis', autospec=True)
@ -156,13 +156,13 @@ class MainTestCase(base.TestCase):
self.root.get_chassis('fake-chassis-id')
mock_chassis.assert_called_once_with(
self.root._conn, 'fake-chassis-id',
self.root.redfish_version, self.root.registries)
self.root.redfish_version, self.root.lazy_registries)
@mock.patch.object(chassis, 'ChassisCollection', autospec=True)
@mock.patch.object(chassis, 'Chassis', autospec=True)
@mock.patch('sushy.Sushy.registries', autospec=True)
@mock.patch('sushy.Sushy.lazy_registries', autospec=True)
def test_get_chassis_default_ok(
self, mock_registries, mock_chassis, mock_chassis_collection):
self, mock_lazy_registries, mock_chassis, mock_chassis_collection):
self.root._standard_message_registries_path = None
mock_chassis.path = 'fake-chassis-id'
mock_members = mock_chassis_collection.return_value.get_members
@ -171,19 +171,19 @@ class MainTestCase(base.TestCase):
mock_chassis_collection.assert_called_once_with(
self.root._conn, '/redfish/v1/Chassis',
redfish_version=self.root.redfish_version,
registries=mock_registries
registries=mock_lazy_registries
)
mock_chassis.assert_called_once_with(
self.root._conn, 'fake-chassis-id',
redfish_version=self.root.redfish_version,
registries=mock_registries
registries=mock_lazy_registries
)
@mock.patch.object(chassis, 'ChassisCollection', autospec=True)
@mock.patch.object(chassis, 'Chassis', autospec=True)
@mock.patch('sushy.Sushy.registries', autospec=True)
@mock.patch('sushy.Sushy.lazy_registries', autospec=True)
def test_get_chassis_default_failure(
self, mock_registries, mock_chassis, mock_chassis_collection):
self, mock_lazy_registries, mock_chassis, mock_chassis_collection):
self.root._standard_message_registries_path = None
mock_members = mock_chassis_collection.return_value.get_members
mock_members.return_value = []
@ -192,7 +192,7 @@ class MainTestCase(base.TestCase):
mock_chassis_collection.assert_called_once_with(
self.root._conn, '/redfish/v1/Chassis',
redfish_version=self.root.redfish_version,
registries=mock_registries
registries=mock_lazy_registries
)
@mock.patch.object(chassis, 'ChassisCollection', autospec=True)
@ -200,41 +200,41 @@ class MainTestCase(base.TestCase):
self.root.get_chassis_collection()
chassis_collection_mock.assert_called_once_with(
self.root._conn, '/redfish/v1/Chassis',
self.root.redfish_version, self.root.registries)
self.root.redfish_version, self.root.lazy_registries)
@mock.patch.object(fabric, 'Fabric', autospec=True)
def test_get_fabric(self, mock_fabric):
self.root.get_fabric('fake-fabric-id')
mock_fabric.assert_called_once_with(
self.root._conn, 'fake-fabric-id',
self.root.redfish_version, self.root.registries)
self.root.redfish_version, self.root.lazy_registries)
@mock.patch.object(fabric, 'FabricCollection', autospec=True)
def test_get_fabric_collection(self, fabric_collection_mock):
self.root.get_fabric_collection()
fabric_collection_mock.assert_called_once_with(
self.root._conn, '/redfish/v1/Fabrics',
self.root.redfish_version, self.root.registries)
self.root.redfish_version, self.root.lazy_registries)
@mock.patch.object(manager, 'ManagerCollection', autospec=True)
def test_get_manager_collection(self, ManagerCollection_mock):
self.root.get_manager_collection()
ManagerCollection_mock.assert_called_once_with(
self.root._conn, '/redfish/v1/Managers',
self.root.redfish_version, self.root.registries)
self.root.redfish_version, self.root.lazy_registries)
@mock.patch.object(manager, 'Manager', autospec=True)
def test_get_manager(self, Manager_mock):
self.root.get_manager('fake-manager-id')
Manager_mock.assert_called_once_with(
self.root._conn, 'fake-manager-id',
self.root.redfish_version, self.root.registries)
self.root.redfish_version, self.root.lazy_registries)
@mock.patch.object(manager, 'ManagerCollection', autospec=True)
@mock.patch.object(manager, 'Manager', autospec=True)
@mock.patch('sushy.Sushy.registries', autospec=True)
@mock.patch('sushy.Sushy.lazy_registries', autospec=True)
def test_get_manager_default_ok(
self, mock_registries, mock_manager, mock_manager_collection):
self, mock_lazy_registries, mock_manager, mock_manager_collection):
self.root._standard_message_registries_path = None
mock_manager.path = 'fake-manager-id'
mock_members = mock_manager_collection.return_value.get_members
@ -243,18 +243,18 @@ class MainTestCase(base.TestCase):
mock_manager_collection.assert_called_once_with(
self.root._conn, '/redfish/v1/Managers',
redfish_version=self.root.redfish_version,
registries=mock_registries
registries=mock_lazy_registries
)
mock_manager.assert_called_once_with(
self.root._conn, 'fake-manager-id',
redfish_version=self.root.redfish_version,
registries=mock_registries)
registries=mock_lazy_registries)
@mock.patch.object(manager, 'ManagerCollection', autospec=True)
@mock.patch.object(manager, 'Manager', autospec=True)
@mock.patch('sushy.Sushy.registries', autospec=True)
@mock.patch('sushy.Sushy.lazy_registries', autospec=True)
def test_get_manager_default_failure(
self, mock_registries, mock_manager, mock_system_collection):
self, mock_lazy_registries, mock_manager, mock_system_collection):
self.root._standard_message_registries_path = None
mock_members = mock_system_collection.return_value.get_members
mock_members.return_value = []
@ -263,7 +263,7 @@ class MainTestCase(base.TestCase):
mock_system_collection.assert_called_once_with(
self.root._conn, '/redfish/v1/Managers',
redfish_version=self.root.redfish_version,
registries=mock_registries
registries=mock_lazy_registries
)
@mock.patch.object(sessionservice, 'SessionService', autospec=True)
@ -278,14 +278,14 @@ class MainTestCase(base.TestCase):
self.root.get_session('asdf')
mock_sess.assert_called_once_with(
self.root._conn, 'asdf',
self.root.redfish_version, self.root.registries)
self.root.redfish_version, self.root.lazy_registries)
@mock.patch.object(updateservice, 'UpdateService', autospec=True)
def test_get_update_service(self, mock_upd_serv):
self.root.get_update_service()
mock_upd_serv.assert_called_once_with(
self.root._conn, '/redfish/v1/UpdateService',
self.root.redfish_version, self.root.registries)
self.root.redfish_version, self.root.lazy_registries)
@mock.patch.object(message_registry_file,
'MessageRegistryFileCollection',
@ -303,7 +303,7 @@ class MainTestCase(base.TestCase):
self.root.get_composition_service()
mock_comp_ser.assert_called_once_with(
self.root._conn, '/redfish/v1/CompositionService',
self.root.redfish_version, self.root.registries)
self.root.redfish_version, self.root.lazy_registries)
def test__get_standard_message_registry_collection(self):
registries = self.root._get_standard_message_registry_collection()
@ -383,6 +383,13 @@ class MainTestCase(base.TestCase):
registries = self.root.registries
self.assertEqual({'RegistryA.2.0': mock_msg_reg1}, registries)
@mock.patch('sushy.Sushy.registries', autospec=True)
def test_lazy_registries(self, mock_registries):
registries = self.root.lazy_registries
self.assertEqual(0, mock_registries.__getitem__.call_count)
registries[1]
self.assertEqual(1, mock_registries.__getitem__.call_count)
class BareMinimumMainTestCase(base.TestCase):