Optimize get_ports with trunk extension

Use case: many trunk ports (300+) each with many subports.
Issue: get_ports takes much time.
Solution: apply trunk resource extend func for a full list of ports,
not for each port individually, thus avoiding a DB query for each
trunk port.

The approach will be extended for other resource extenders, like QoS,
and probably for other resources as well.
QoS: https://review.opendev.org/c/openstack/neutron/+/764454

Change-Id: Ic08e4049f6156c0700ca3c7aee251b6eb0eb97da
Closes-Bug: #1905268
(cherry picked from commit e007349d47)

neutron/db/db_base_plugin_common.py

@@ -217,7 +217,8 @@ class DbBasePluginCommon(object):
 
     def _make_port_dict(self, port, fields=None,
                         process_extensions=True,
-                        with_fixed_ips=True):
+                        with_fixed_ips=True,
+                        bulk=False):
         mac = port["mac_address"]
         if isinstance(mac, netaddr.EUI):
             mac.dialect = netaddr.mac_unix_expanded
@@ -240,8 +241,10 @@ class DbBasePluginCommon(object):
             port_data = port
             if isinstance(port, port_obj.Port):
                 port_data = port.db_obj
+            res['bulk'] = bulk
             resource_extend.apply_funcs(
                 port_def.COLLECTION_NAME, res, port_data)
+            res.pop('bulk')
         return db_utils.resource_fields(res, fields)
 
     def _get_network(self, context, id):

neutron/db/db_base_plugin_v2.py

@@ -1570,7 +1570,9 @@ class NeutronDbPluginV2(db_base_plugin_common.DbBasePluginCommon,
                                       sorts=sorts, limit=limit,
                                       marker_obj=marker_obj,
                                       page_reverse=page_reverse)
-        items = [self._make_port_dict(c, fields) for c in query]
+        items = [self._make_port_dict(c, fields, bulk=True) for c in query]
+        # TODO(obondarev): use neutron_lib constant
+        resource_extend.apply_funcs('ports_bulk', items, None)
         if limit and page_reverse:
             items.reverse()
         return items

neutron/services/trunk/plugin.py

@@ -79,17 +79,44 @@ class TrunkPlugin(service_base.ServicePluginBase):
                             'port_id': x.port_id}
                 for x in port_db.trunk_port.sub_ports
             }
-            core_plugin = directory.get_plugin()
+            if not port_res.get('bulk'):
-            ports = core_plugin.get_ports(
+                core_plugin = directory.get_plugin()
-                context.get_admin_context(), filters={'id': subports})
+                ports = core_plugin.get_ports(
-            for port in ports:
+                    context.get_admin_context(), filters={'id': subports})
-                subports[port['id']]['mac_address'] = port['mac_address']
+                for port in ports:
+                    subports[port['id']]['mac_address'] = port['mac_address']
             trunk_details = {'trunk_id': port_db.trunk_port.id,
                              'sub_ports': [x for x in subports.values()]}
             port_res['trunk_details'] = trunk_details
 
         return port_res
 
+    @staticmethod
+    # TODO(obondarev): use neutron_lib constant
+    @resource_extend.extends(['ports_bulk'])
+    def _extend_port_trunk_details_bulk(ports_res, noop):
+        """Add trunk subport details to a list of ports."""
+        subport_ids = []
+        trunk_ports = []
+        for p in ports_res:
+            if 'trunk_details' in p and 'subports' in p['trunk_details']:
+                trunk_ports.append(p)
+                for subp in p['trunk_details']['subports']:
+                    subport_ids.append(subp['port_id'])
+        if not subport_ids:
+            return ports_res
+
+        core_plugin = directory.get_plugin()
+        subports = core_plugin.get_ports(
+            context.get_admin_context(), filters={'id': subport_ids})
+        subport_macs = {p['id']: p['mac_address'] for p in subports}
+
+        for tp in trunk_ports:
+            for subp in tp['trunk_details']['subports']:
+                subp['mac_address'] = subport_macs[subp['port_id']]
+
+        return ports_res
+
     def check_compatibility(self):
         """Verify the plugin can load correctly and fail otherwise."""
         self.check_driver_compatibility()