Batch db segment retrieval

A net-list operation was calling extend_network_dict_provider for
each network individually which would result in a database call for
each network.

This adds a new call in the manager to extend multiple networks at
once and then it adds a bulk version of get_network_segments that
it calls.

Now 1 net list of any number of networks will only result in 1
segment DB call.

Change-Id: I2543b3bdbb178ee4bb8d1288e9a27af1c5c8c8b4
Closes-Bug: #1525423
Partial-Bug: #1513782

neutron/plugins/ml2/db.py

@@ -65,15 +65,22 @@ def add_network_segment(session, network_id, segment, segment_index=0,
 
 
 def get_network_segments(session, network_id, filter_dynamic=False):
+    return get_networks_segments(
+        session, [network_id], filter_dynamic)[network_id]
+
+
+def get_networks_segments(session, network_ids, filter_dynamic=False):
     with session.begin(subtransactions=True):
         query = (session.query(models.NetworkSegment).
-                 filter_by(network_id=network_id).
+                 filter(models.NetworkSegment.network_id.in_(network_ids)).
                  order_by(models.NetworkSegment.segment_index))
         if filter_dynamic is not None:
             query = query.filter_by(is_dynamic=filter_dynamic)
         records = query.all()
-
-        return [_make_segment_dict(record) for record in records]
+        result = {net_id: [] for net_id in network_ids}
+        for record in records:
+            result[record.network_id].append(_make_segment_dict(record))
+        return result
 
 
 def get_segment_by_id(session, segment_id):

neutron/plugins/ml2/managers.py

@@ -148,10 +148,20 @@ class TypeManager(stevedore.named.NamedExtensionManager):
         return value
 
     def extend_network_dict_provider(self, context, network):
-        id = network['id']
-        segments = db.get_network_segments(context.session, id)
+        # this method is left for backward compat even though it would be
+        # easy to change the callers in tree to use the bulk function
+        return self.extend_networks_dict_provider(context, [network])
+
+    def extend_networks_dict_provider(self, context, networks):
+        ids = [network['id'] for network in networks]
+        net_segments = db.get_networks_segments(context.session, ids)
+        for network in networks:
+            segments = net_segments[network['id']]
+            self._extend_network_dict_provider(network, segments)
+
+    def _extend_network_dict_provider(self, network, segments):
         if not segments:
-            LOG.error(_LE("Network %s has no segments"), id)
+            LOG.error(_LE("Network %s has no segments"), network['id'])
             for attr in provider.ATTRIBUTES:
                 network[attr] = None
         elif len(segments) > 1:

neutron/plugins/ml2/plugin.py

@@ -722,8 +722,7 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2,
             nets = super(Ml2Plugin,
                          self).get_networks(context, filters, None, sorts,
                                             limit, marker, page_reverse)
-            for net in nets:
-                self.type_manager.extend_network_dict_provider(context, net)
+            self.type_manager.extend_networks_dict_provider(context, nets)
 
             nets = self._filter_nets_provider(context, nets, filters)
 

neutron/tests/unit/plugins/ml2/test_db.py

@@ -57,8 +57,8 @@ class Ml2DBTestCase(testlib_api.SqlTestCase):
                                                     vif_type=vif_type,
                                                     host=host))
 
-    def _create_segments(self, segments, is_seg_dynamic=False):
-        network_id = 'foo-network-id'
+    def _create_segments(self, segments, is_seg_dynamic=False,
+                         network_id='foo-network-id'):
         self._setup_neutron_network(network_id)
         for segment in segments:
             ml2_db.add_network_segment(
@@ -95,6 +95,32 @@ class Ml2DBTestCase(testlib_api.SqlTestCase):
                      api.SEGMENTATION_ID: 2}]
         self._create_segments(segments)
 
+    def test_get_networks_segments(self):
+        segments1 = [{api.NETWORK_TYPE: 'vlan',
+                      api.PHYSICAL_NETWORK: 'physnet1',
+                      api.SEGMENTATION_ID: 1},
+                     {api.NETWORK_TYPE: 'vlan',
+                      api.PHYSICAL_NETWORK: 'physnet1',
+                      api.SEGMENTATION_ID: 2}]
+        segments2 = [{api.NETWORK_TYPE: 'vlan',
+                      api.PHYSICAL_NETWORK: 'physnet1',
+                      api.SEGMENTATION_ID: 3},
+                     {api.NETWORK_TYPE: 'vlan',
+                      api.PHYSICAL_NETWORK: 'physnet1',
+                      api.SEGMENTATION_ID: 4}]
+        net1segs = self._create_segments(segments1, network_id='net1')
+        net2segs = self._create_segments(segments2, network_id='net2')
+        segs = ml2_db.get_networks_segments(self.ctx.session, ['net1', 'net2'])
+        self.assertEqual(net1segs, segs['net1'])
+        self.assertEqual(net2segs, segs['net2'])
+
+    def test_get_networks_segments_no_segments(self):
+        self._create_segments([], network_id='net1')
+        self._create_segments([], network_id='net2')
+        segs = ml2_db.get_networks_segments(self.ctx.session, ['net1', 'net2'])
+        self.assertEqual([], segs['net1'])
+        self.assertEqual([], segs['net2'])
+
     def test_get_segment_by_id(self):
         segment = {api.NETWORK_TYPE: 'vlan',
                    api.PHYSICAL_NETWORK: 'physnet1',

neutron/tests/unit/plugins/ml2/test_plugin.py

@@ -676,10 +676,10 @@ class TestMl2PortsV2(test_plugin.TestPortsV2, Ml2PluginV2TestCase):
         ctx = context.get_admin_context()
         with self.network() as net:
             with self.subnet(network=net) as subnet:
-                segments = ml2_db.get_network_segments(ctx.session,
-                                                       net['network']['id'])
+                segments = ml2_db.get_networks_segments(ctx.session,
+                                                        [net['network']['id']])
                 with mock.patch('neutron.plugins.ml2.plugin.'
-                                'db.get_network_segments') as get_seg_mock:
+                                'db.get_networks_segments') as get_seg_mock:
                     get_seg_mock.side_effect = [db_exc.DBDeadlock, segments,
                                                 segments, segments]
                     with self.port(subnet=subnet) as port: