Improve VLAN allocations synchronization

In order to reduce the number of elements retrieved from the DB, this
patch, before processing the VLAN allocations per physical network,
deleted those registers belonging to any unconfigured physical network.

The VLAN registers per physical network are deleted using a bulk delete
operation, to speed up the process.

Those missing VLAN registers per network are now created using a bulk
insert operation, available in the ORM. This bulk operation speeds up
the sync process.

Change-Id: I8568e2277e157754aaff87a059a40e34e6a43e2b
Partial-Bug: #1862178

neutron/objects/plugins/ml2/vlanallocation.py

@@ -33,3 +33,24 @@ class VlanAllocation(base.NeutronDbObject):
     }
 
     primary_keys = ['physical_network', 'vlan_id']
+
+    @staticmethod
+    def get_physical_networks(context):
+        query = context.session.query(VlanAllocation.db_model.physical_network)
+        query = query.group_by(VlanAllocation.db_model.physical_network)
+        physnets = query.all()
+        return {physnet.physical_network for physnet in physnets}
+
+    @staticmethod
+    def delete_physical_networks(context, physical_networks):
+        column = VlanAllocation.db_model.physical_network
+        context.session.query(VlanAllocation.db_model).filter(
+            column.in_(physical_networks)).delete(synchronize_session=False)
+
+    @staticmethod
+    def bulk_create(ctx, physical_network, vlan_ids):
+        ctx.session.bulk_insert_mappings(
+            vlan_alloc_model.VlanAllocation,
+            [{'physical_network': physical_network,
+              'allocated': False,
+              'vlan_id': vlan_id} for vlan_id in vlan_ids])

neutron/plugins/ml2/drivers/type_vlan.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import collections
 import sys
 
 from neutron_lib import constants as p_const
@@ -26,7 +27,6 @@ from neutron_lib.plugins import utils as plugin_utils
 from oslo_config import cfg
 from oslo_log import log
 from oslo_utils import uuidutils
-from six import moves
 
 from neutron._i18n import _
 from neutron.conf.plugins.ml2.drivers import driver_type
@@ -104,23 +104,30 @@ class VlanTypeDriver(helpers.SegmentTypeDriver):
     def _sync_vlan_allocations(self):
         ctx = context.get_admin_context()
         with db_api.CONTEXT_WRITER.using(ctx):
-            # get existing allocations for all physical networks
-            allocations = dict()
-            allocs = vlanalloc.VlanAllocation.get_objects(ctx)
-            for alloc in allocs:
-                if alloc.physical_network not in allocations:
-                    allocations[alloc.physical_network] = list()
+            # VLAN ranges per physical network:
+            #   {phy1: [(1, 10), (30, 50)], ...}
+            ranges = self.get_network_segment_ranges()
+
+            # Delete those VLAN registers from unconfigured physical networks
+            physnets = vlanalloc.VlanAllocation.get_physical_networks(ctx)
+            physnets_unconfigured = physnets - set(ranges)
+            if physnets_unconfigured:
+                LOG.debug('Removing any VLAN register on physical networks %s',
+                          physnets_unconfigured)
+                vlanalloc.VlanAllocation.delete_physical_networks(
+                    ctx, physnets_unconfigured)
+
+            # Get existing allocations for all configured physical networks
+            allocations = collections.defaultdict(list)
+            for alloc in vlanalloc.VlanAllocation.get_objects(ctx):
                 allocations[alloc.physical_network].append(alloc)
 
-            # process vlan ranges for each configured physical network
-            ranges = self.get_network_segment_ranges()
-            for (physical_network,
-                 vlan_ranges) in ranges.items():
+            for physical_network, vlan_ranges in ranges.items():
                 # determine current configured allocatable vlans for
                 # this physical network
                 vlan_ids = set()
                 for vlan_min, vlan_max in vlan_ranges:
-                    vlan_ids |= set(moves.range(vlan_min, vlan_max + 1))
+                    vlan_ids |= set(range(vlan_min, vlan_max + 1))
 
                 # remove from table unallocated vlans not currently
                 # allocatable
@@ -153,25 +160,9 @@ class VlanTypeDriver(helpers.SegmentTypeDriver):
                                     alloc.delete()
                     del allocations[physical_network]
 
-                # add missing allocatable vlans to table
-                for vlan_id in sorted(vlan_ids):
-                    alloc = vlanalloc.VlanAllocation(
-                        ctx,
-                        physical_network=physical_network,
-                        vlan_id=vlan_id, allocated=False)
-                    alloc.create()
-
-            # remove from table unallocated vlans for any unconfigured
-            # physical networks
-            for allocs in allocations.values():
-                for alloc in allocs:
-                    if not alloc.allocated:
-                        LOG.debug("Removing vlan %(vlan_id)s on physical "
-                                  "network %(physical_network)s from pool",
-                                  {'vlan_id': alloc.vlan_id,
-                                   'physical_network':
-                                   alloc.physical_network})
-                        alloc.delete()
+                # Add missing allocatable VLAN registers for "physical_network"
+                vlanalloc.VlanAllocation.bulk_create(ctx, physical_network,
+                                                     vlan_ids)
 
     @db_api.retry_db_errors
     def _get_network_segment_ranges_from_db(self):

neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py

@@ -30,6 +30,7 @@ from neutron.tests.unit import testlib_api
 
 PROVIDER_NET = 'phys_net1'
 TENANT_NET = 'phys_net2'
+UNCONFIGURED_NET = 'no_net'
 VLAN_MIN = 200
 VLAN_MAX = 209
 NETWORK_VLAN_RANGES = [PROVIDER_NET, "%s:%s:%s" %
@@ -41,6 +42,12 @@ UPDATED_VLAN_RANGES = {
 EMPTY_VLAN_RANGES = {
     PROVIDER_NET: []
 }
+NETWORK_VLAN_RANGES_WITH_UNCONFIG = {
+    PROVIDER_NET: [],
+    TENANT_NET: [(VLAN_MIN + 5, VLAN_MAX + 5)],
+    UNCONFIGURED_NET: [(VLAN_MIN, VLAN_MAX)]
+}
+
 CORE_PLUGIN = 'ml2'
 SERVICE_PLUGIN_KLASS = ('neutron.services.network_segment_range.plugin.'
                         'NetworkSegmentRangePlugin')
@@ -171,10 +178,21 @@ class VlanTypeTest(testlib_api.SqlTestCase):
                 self._get_allocation(self.context, segment).allocated)
 
         check_in_ranges(self.network_vlan_ranges)
+
         self.driver.network_vlan_ranges = UPDATED_VLAN_RANGES
         self.driver._sync_vlan_allocations()
         check_in_ranges(UPDATED_VLAN_RANGES)
 
+        self.driver.network_vlan_ranges = NETWORK_VLAN_RANGES_WITH_UNCONFIG
+        self.driver._sync_vlan_allocations()
+        self.driver.network_vlan_ranges = UPDATED_VLAN_RANGES
+        with mock.patch.object(type_vlan.LOG, 'debug') as mock_debug:
+            self.driver._sync_vlan_allocations()
+            mock_debug.assert_called_once_with(
+                'Removing any VLAN register on physical networks %s',
+                {UNCONFIGURED_NET})
+        check_in_ranges(UPDATED_VLAN_RANGES)
+
         self.driver.network_vlan_ranges = EMPTY_VLAN_RANGES
         self.driver._sync_vlan_allocations()