From cb9d2ba2e70d57abf12ba86c6cb16679ae567270 Mon Sep 17 00:00:00 2001 From: Oleg Bondarev Date: Mon, 23 Nov 2020 13:05:15 +0400 Subject: [PATCH] Optimize get_ports with trunk extension Use case: many trunk ports (300+) each with many subports. Issue: get_ports takes much time. Solution: apply trunk resource extend func for a full list of ports, not for each port individually, thus avoiding a DB query for each trunk port. The approach will be extended for other resource extenders, like QoS, and probably for other resources as well. QoS: https://review.opendev.org/c/openstack/neutron/+/764454 Change-Id: Ic08e4049f6156c0700ca3c7aee251b6eb0eb97da Closes-Bug: #1905268 (cherry picked from commit e007349d47812f2307e8951ed44f3c392089bb3b) --- neutron/db/db_base_plugin_common.py | 5 +++- neutron/db/db_base_plugin_v2.py | 4 +++- neutron/services/trunk/plugin.py | 37 +++++++++++++++++++++++++---- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/neutron/db/db_base_plugin_common.py b/neutron/db/db_base_plugin_common.py index 21e895aa2a3..164943ded21 100644 --- a/neutron/db/db_base_plugin_common.py +++ b/neutron/db/db_base_plugin_common.py @@ -217,7 +217,8 @@ class DbBasePluginCommon(object): def _make_port_dict(self, port, fields=None, process_extensions=True, - with_fixed_ips=True): + with_fixed_ips=True, + bulk=False): mac = port["mac_address"] if isinstance(mac, netaddr.EUI): mac.dialect = netaddr.mac_unix_expanded @@ -240,8 +241,10 @@ class DbBasePluginCommon(object): port_data = port if isinstance(port, port_obj.Port): port_data = port.db_obj + res['bulk'] = bulk resource_extend.apply_funcs( port_def.COLLECTION_NAME, res, port_data) + res.pop('bulk') return db_utils.resource_fields(res, fields) def _get_network(self, context, id): diff --git a/neutron/db/db_base_plugin_v2.py b/neutron/db/db_base_plugin_v2.py index 42ce58e168a..392cd5d0bc6 100644 --- a/neutron/db/db_base_plugin_v2.py +++ b/neutron/db/db_base_plugin_v2.py @@ -1570,7 +1570,9 @@ class NeutronDbPluginV2(db_base_plugin_common.DbBasePluginCommon, sorts=sorts, limit=limit, marker_obj=marker_obj, page_reverse=page_reverse) - items = [self._make_port_dict(c, fields) for c in query] + items = [self._make_port_dict(c, fields, bulk=True) for c in query] + # TODO(obondarev): use neutron_lib constant + resource_extend.apply_funcs('ports_bulk', items, None) if limit and page_reverse: items.reverse() return items diff --git a/neutron/services/trunk/plugin.py b/neutron/services/trunk/plugin.py index 63820dcb5b2..ad150f3db53 100644 --- a/neutron/services/trunk/plugin.py +++ b/neutron/services/trunk/plugin.py @@ -79,17 +79,44 @@ class TrunkPlugin(service_base.ServicePluginBase): 'port_id': x.port_id} for x in port_db.trunk_port.sub_ports } - core_plugin = directory.get_plugin() - ports = core_plugin.get_ports( - context.get_admin_context(), filters={'id': subports}) - for port in ports: - subports[port['id']]['mac_address'] = port['mac_address'] + if not port_res.get('bulk'): + core_plugin = directory.get_plugin() + ports = core_plugin.get_ports( + context.get_admin_context(), filters={'id': subports}) + for port in ports: + subports[port['id']]['mac_address'] = port['mac_address'] trunk_details = {'trunk_id': port_db.trunk_port.id, 'sub_ports': [x for x in subports.values()]} port_res['trunk_details'] = trunk_details return port_res + @staticmethod + # TODO(obondarev): use neutron_lib constant + @resource_extend.extends(['ports_bulk']) + def _extend_port_trunk_details_bulk(ports_res, noop): + """Add trunk subport details to a list of ports.""" + subport_ids = [] + trunk_ports = [] + for p in ports_res: + if 'trunk_details' in p and 'subports' in p['trunk_details']: + trunk_ports.append(p) + for subp in p['trunk_details']['subports']: + subport_ids.append(subp['port_id']) + if not subport_ids: + return ports_res + + core_plugin = directory.get_plugin() + subports = core_plugin.get_ports( + context.get_admin_context(), filters={'id': subport_ids}) + subport_macs = {p['id']: p['mac_address'] for p in subports} + + for tp in trunk_ports: + for subp in tp['trunk_details']['subports']: + subp['mac_address'] = subport_macs[subp['port_id']] + + return ports_res + def check_compatibility(self): """Verify the plugin can load correctly and fail otherwise.""" self.check_driver_compatibility()