From 89c56d738df62fd4806e958679cfd64fe2f29631 Mon Sep 17 00:00:00 2001 From: Rodolfo Alonso Hernandez Date: Tue, 11 Feb 2020 14:09:06 +0000 Subject: [PATCH] Improve VLAN allocations synchronization In order to reduce the number of elements retrieved from the DB, this patch, before processing the VLAN allocations per physical network, deleted those registers belonging to any unconfigured physical network. The VLAN registers per physical network are deleted using a bulk delete operation, to speed up the process. Those missing VLAN registers per network are now created using a bulk insert operation, available in the ORM. This bulk operation speeds up the sync process. Conflicts: neutron/plugins/ml2/drivers/type_vlan.py Change-Id: I8568e2277e157754aaff87a059a40e34e6a43e2b Partial-Bug: #1862178 (cherry picked from commit 016e7826f165e69c3956375fc3aa8b8d642c9dc9) (cherry picked from commit 651eb12bec5e5e986bed79f3a5c006c617e79bda) (cherry picked from commit 4fff732b76ee5d9a4917901b33ce80ccafb5203f) (cherry picked from commit 8a6521e4457acbd7ceed941f363ee7c9872422ba) --- neutron/objects/plugins/ml2/vlanallocation.py | 21 ++++++++ neutron/plugins/ml2/drivers/type_vlan.py | 52 ++++++++----------- .../plugins/ml2/drivers/test_type_vlan.py | 18 +++++++ 3 files changed, 61 insertions(+), 30 deletions(-) diff --git a/neutron/objects/plugins/ml2/vlanallocation.py b/neutron/objects/plugins/ml2/vlanallocation.py index 402e316b873..16944bc9591 100644 --- a/neutron/objects/plugins/ml2/vlanallocation.py +++ b/neutron/objects/plugins/ml2/vlanallocation.py @@ -33,3 +33,24 @@ class VlanAllocation(base.NeutronDbObject): } primary_keys = ['physical_network', 'vlan_id'] + + @staticmethod + def get_physical_networks(context): + query = context.session.query(VlanAllocation.db_model.physical_network) + query = query.group_by(VlanAllocation.db_model.physical_network) + physnets = query.all() + return {physnet.physical_network for physnet in physnets} + + @staticmethod + def delete_physical_networks(context, physical_networks): + column = VlanAllocation.db_model.physical_network + context.session.query(VlanAllocation.db_model).filter( + column.in_(physical_networks)).delete(synchronize_session=False) + + @staticmethod + def bulk_create(ctx, physical_network, vlan_ids): + ctx.session.bulk_insert_mappings( + vlan_alloc_model.VlanAllocation, + [{'physical_network': physical_network, + 'allocated': False, + 'vlan_id': vlan_id} for vlan_id in vlan_ids]) diff --git a/neutron/plugins/ml2/drivers/type_vlan.py b/neutron/plugins/ml2/drivers/type_vlan.py index 6b6831dcf00..e746f26048c 100644 --- a/neutron/plugins/ml2/drivers/type_vlan.py +++ b/neutron/plugins/ml2/drivers/type_vlan.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import collections import sys from neutron_lib import constants as p_const @@ -21,7 +22,6 @@ from neutron_lib import exceptions as exc from neutron_lib.plugins.ml2 import api from oslo_config import cfg from oslo_log import log -from six import moves from neutron._i18n import _ from neutron.conf.plugins.ml2.drivers import driver_type @@ -64,22 +64,30 @@ class VlanTypeDriver(helpers.SegmentTypeDriver): def _sync_vlan_allocations(self): ctx = context.get_admin_context() with db_api.context_manager.writer.using(ctx): - # get existing allocations for all physical networks - allocations = dict() - allocs = vlanalloc.VlanAllocation.get_objects(ctx) - for alloc in allocs: - if alloc.physical_network not in allocations: - allocations[alloc.physical_network] = list() + # VLAN ranges per physical network: + # {phy1: [(1, 10), (30, 50)], ...} + ranges = self.network_vlan_ranges + + # Delete those VLAN registers from unconfigured physical networks + physnets = vlanalloc.VlanAllocation.get_physical_networks(ctx) + physnets_unconfigured = physnets - set(ranges) + if physnets_unconfigured: + LOG.debug('Removing any VLAN register on physical networks %s', + physnets_unconfigured) + vlanalloc.VlanAllocation.delete_physical_networks( + ctx, physnets_unconfigured) + + # Get existing allocations for all configured physical networks + allocations = collections.defaultdict(list) + for alloc in vlanalloc.VlanAllocation.get_objects(ctx): allocations[alloc.physical_network].append(alloc) - # process vlan ranges for each configured physical network - for (physical_network, - vlan_ranges) in self.network_vlan_ranges.items(): + for physical_network, vlan_ranges in ranges.items(): # determine current configured allocatable vlans for # this physical network vlan_ids = set() for vlan_min, vlan_max in vlan_ranges: - vlan_ids |= set(moves.range(vlan_min, vlan_max + 1)) + vlan_ids |= set(range(vlan_min, vlan_max + 1)) # remove from table unallocated vlans not currently # allocatable @@ -112,25 +120,9 @@ class VlanTypeDriver(helpers.SegmentTypeDriver): alloc.delete() del allocations[physical_network] - # add missing allocatable vlans to table - for vlan_id in sorted(vlan_ids): - alloc = vlanalloc.VlanAllocation( - ctx, - physical_network=physical_network, - vlan_id=vlan_id, allocated=False) - alloc.create() - - # remove from table unallocated vlans for any unconfigured - # physical networks - for allocs in allocations.values(): - for alloc in allocs: - if not alloc.allocated: - LOG.debug("Removing vlan %(vlan_id)s on physical " - "network %(physical_network)s from pool", - {'vlan_id': alloc.vlan_id, - 'physical_network': - alloc.physical_network}) - alloc.delete() + # Add missing allocatable VLAN registers for "physical_network" + vlanalloc.VlanAllocation.bulk_create(ctx, physical_network, + vlan_ids) def get_type(self): return p_const.TYPE_VLAN diff --git a/neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py b/neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py index 93581716f90..9b18164f5ea 100644 --- a/neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py +++ b/neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py @@ -29,6 +29,7 @@ from neutron.tests.unit import testlib_api PROVIDER_NET = 'phys_net1' TENANT_NET = 'phys_net2' +UNCONFIGURED_NET = 'no_net' VLAN_MIN = 200 VLAN_MAX = 209 NETWORK_VLAN_RANGES = [PROVIDER_NET, "%s:%s:%s" % @@ -40,6 +41,12 @@ UPDATED_VLAN_RANGES = { EMPTY_VLAN_RANGES = { PROVIDER_NET: [] } +NETWORK_VLAN_RANGES_WITH_UNCONFIG = { + PROVIDER_NET: [], + TENANT_NET: [(VLAN_MIN + 5, VLAN_MAX + 5)], + UNCONFIGURED_NET: [(VLAN_MIN, VLAN_MAX)] +} + CORE_PLUGIN = 'ml2' @@ -168,10 +175,21 @@ class VlanTypeTest(testlib_api.SqlTestCase): self._get_allocation(self.context, segment).allocated) check_in_ranges(self.network_vlan_ranges) + self.driver.network_vlan_ranges = UPDATED_VLAN_RANGES self.driver._sync_vlan_allocations() check_in_ranges(UPDATED_VLAN_RANGES) + self.driver.network_vlan_ranges = NETWORK_VLAN_RANGES_WITH_UNCONFIG + self.driver._sync_vlan_allocations() + self.driver.network_vlan_ranges = UPDATED_VLAN_RANGES + with mock.patch.object(type_vlan.LOG, 'debug') as mock_debug: + self.driver._sync_vlan_allocations() + mock_debug.assert_called_once_with( + 'Removing any VLAN register on physical networks %s', + {UNCONFIGURED_NET}) + check_in_ranges(UPDATED_VLAN_RANGES) + self.driver.network_vlan_ranges = EMPTY_VLAN_RANGES self.driver._sync_vlan_allocations()