From bbe401aaf9bfdd77e1d43d547b2cdb436b1440c8 Mon Sep 17 00:00:00 2001 From: Rodolfo Alonso Hernandez Date: Wed, 26 Feb 2020 10:39:19 +0000 Subject: [PATCH] Fix queries to retrieve allocations with network_segment_range Fixed the queries to retrieve the segment ID allocations when service plugin network_segment_range is enabled. With the previous implementation, a project user was able to allocate a segment ID belonging to other project segment range. The solution implemented was discussed in [1]: - A project user will retrieve segments from the project ranges. - When depleted, the segment IDs will be retrieved from the shared range, never using another project segment ID. [1]http://lists.openstack.org/pipermail/openstack-discuss/2020-February/012736.html Conflicts: neutron/objects/network_segment_range.py neutron/objects/plugins/ml2/base.py neutron/objects/plugins/ml2/vlanallocation.py neutron/objects/plugins/ml2/vxlanallocation.py Change-Id: I953062d9ee8ee5ee9a9f07aff4a8222ac63ed525 Closes-Bug: #1863423 (cherry picked from commit 046672247de56bad950e8267a57bd26205f354a0) --- neutron/common/_constants.py | 3 + .../db/models/plugins/ml2/geneveallocation.py | 8 + .../plugins/ml2/gre_allocation_endpoints.py | 8 + .../db/models/plugins/ml2/vlanallocation.py | 8 + .../db/models/plugins/ml2/vxlanallocation.py | 8 + neutron/objects/network_segment_range.py | 87 ++++++++++ neutron/objects/plugins/ml2/base.py | 44 ++++- neutron/objects/plugins/ml2/flatallocation.py | 4 +- .../objects/plugins/ml2/geneveallocation.py | 9 +- neutron/objects/plugins/ml2/greallocation.py | 9 +- neutron/objects/plugins/ml2/vlanallocation.py | 16 +- .../objects/plugins/ml2/vxlanallocation.py | 9 +- neutron/plugins/ml2/drivers/helpers.py | 140 ++++------------ .../unit/objects/plugins/ml2/test_base.py | 34 ++++ .../plugins/ml2/test_geneveallocation.py | 6 +- .../objects/plugins/ml2/test_greallocation.py | 6 +- .../plugins/ml2/test_vlanallocation.py | 6 +- .../plugins/ml2/test_vxlanallocation.py | 6 +- .../objects/test_network_segment_range.py | 154 +++++++++++++++--- 19 files changed, 420 insertions(+), 145 deletions(-) create mode 100644 neutron/tests/unit/objects/plugins/ml2/test_base.py diff --git a/neutron/common/_constants.py b/neutron/common/_constants.py index 01fd3ebd124..5673d3af5bb 100644 --- a/neutron/common/_constants.py +++ b/neutron/common/_constants.py @@ -64,3 +64,6 @@ RPC_RES_PROCESSING_STEP = 20 # IPtables version to support --random-fully option. # Do not move this constant to neutron-lib, since it is temporary IPTABLES_RANDOM_FULLY_VERSION = '1.6.2' + +# Segmentation ID pool; DB select limit to improve the performace. +IDPOOL_SELECT_SIZE = 100 diff --git a/neutron/db/models/plugins/ml2/geneveallocation.py b/neutron/db/models/plugins/ml2/geneveallocation.py index 26ca64ffd57..8007b494d9d 100644 --- a/neutron/db/models/plugins/ml2/geneveallocation.py +++ b/neutron/db/models/plugins/ml2/geneveallocation.py @@ -29,6 +29,14 @@ class GeneveAllocation(model_base.BASEV2): def get_segmentation_id(cls): return cls.geneve_vni + @property + def segmentation_id(self): + return self.geneve_vni + + @staticmethod + def primary_keys(): + return {'geneve_vni'} + class GeneveEndpoints(model_base.BASEV2): """Represents tunnel endpoint in RPC mode.""" diff --git a/neutron/db/models/plugins/ml2/gre_allocation_endpoints.py b/neutron/db/models/plugins/ml2/gre_allocation_endpoints.py index 79dee7f14c1..3b75569e74c 100644 --- a/neutron/db/models/plugins/ml2/gre_allocation_endpoints.py +++ b/neutron/db/models/plugins/ml2/gre_allocation_endpoints.py @@ -31,6 +31,14 @@ class GreAllocation(model_base.BASEV2): def get_segmentation_id(cls): return cls.gre_id + @property + def segmentation_id(self): + return self.gre_id + + @staticmethod + def primary_keys(): + return {'gre_id'} + class GreEndpoints(model_base.BASEV2): """Represents tunnel endpoint in RPC mode.""" diff --git a/neutron/db/models/plugins/ml2/vlanallocation.py b/neutron/db/models/plugins/ml2/vlanallocation.py index 60877e82a2d..e36258ce2f2 100644 --- a/neutron/db/models/plugins/ml2/vlanallocation.py +++ b/neutron/db/models/plugins/ml2/vlanallocation.py @@ -43,3 +43,11 @@ class VlanAllocation(model_base.BASEV2): @classmethod def get_segmentation_id(cls): return cls.vlan_id + + @property + def segmentation_id(self): + return self.vlan_id + + @staticmethod + def primary_keys(): + return {'vlan_id', 'physical_network'} diff --git a/neutron/db/models/plugins/ml2/vxlanallocation.py b/neutron/db/models/plugins/ml2/vxlanallocation.py index acdfb21a862..b0ec99f06c7 100644 --- a/neutron/db/models/plugins/ml2/vxlanallocation.py +++ b/neutron/db/models/plugins/ml2/vxlanallocation.py @@ -31,6 +31,14 @@ class VxlanAllocation(model_base.BASEV2): def get_segmentation_id(cls): return cls.vxlan_vni + @property + def segmentation_id(self): + return self.vxlan_vni + + @staticmethod + def primary_keys(): + return {'vxlan_vni'} + class VxlanEndpoints(model_base.BASEV2): """Represents tunnel endpoint in RPC mode.""" diff --git a/neutron/objects/network_segment_range.py b/neutron/objects/network_segment_range.py index dc1106fb2b5..4840abd9b7f 100644 --- a/neutron/objects/network_segment_range.py +++ b/neutron/objects/network_segment_range.py @@ -12,14 +12,21 @@ # License for the specific language governing permissions and limitations # under the License. +import copy +import itertools + from neutron_lib import constants from neutron_lib.db import utils as db_utils from neutron_lib import exceptions as n_exc from oslo_versionedobjects import fields as obj_fields +from six.moves import range as six_range from sqlalchemy import and_ from sqlalchemy import not_ +from sqlalchemy import or_ +from sqlalchemy import sql from neutron._i18n import _ +from neutron.common import _constants as common_constants from neutron.db.models import network_segment_range as range_model from neutron.db.models.plugins.ml2 import geneveallocation as \ geneve_alloc_model @@ -140,3 +147,83 @@ class NetworkSegmentRange(base.NeutronDbObject): models_v2.Network.id)).all() return {segmentation_id: project_id for segmentation_id, project_id in alloc_used} + + @classmethod + def _build_query_segments(cls, context, model, network_type, **filters): + columns = set(dict(model.__table__.columns)) + model_filters = dict((k, filters[k]) + for k in columns & set(filters.keys())) + query = (context.session.query(model) + .filter_by(allocated=False, **model_filters).distinct()) + _and = and_( + cls.db_model.network_type == network_type, + model.physical_network == cls.db_model.physical_network if + network_type == constants.TYPE_VLAN else sql.expression.true()) + return query.join(range_model.NetworkSegmentRange, _and) + + @classmethod + def get_segments_for_project(cls, context, model, network_type, + model_segmentation_id, **filters): + _filters = copy.deepcopy(filters) + project_id = _filters.pop('project_id', None) + if not project_id: + return [] + + with cls.db_context_reader(context): + query = cls._build_query_segments(context, model, network_type, + **_filters) + query = query.filter(and_( + model_segmentation_id >= cls.db_model.minimum, + model_segmentation_id <= cls.db_model.maximum, + cls.db_model.project_id == project_id)) + return query.limit(common_constants.IDPOOL_SELECT_SIZE).all() + + @classmethod + def get_segments_shared(cls, context, model, network_type, + model_segmentation_id, **filters): + _filters = copy.deepcopy(filters) + project_id = _filters.pop('project_id', None) + with cls.db_context_reader(context): + # Retrieve default segment ID range. + default_range = context.session.query(cls.db_model).filter( + and_(cls.db_model.network_type == network_type, + cls.db_model.default == sql.expression.true())) + if network_type == constants.TYPE_VLAN: + default_range.filter(cls.db_model.physical_network == + _filters['physical_network']) + segment_ids = set(six_range(default_range.all()[0].minimum, + default_range.all()[0].maximum + 1)) + + # Retrieve other project segment ID ranges (not own project, not + # default range). + other_project_ranges = context.session.query(cls.db_model).filter( + and_(cls.db_model.project_id != project_id, + cls.db_model.project_id.isnot(None), + cls.db_model.network_type == network_type)) + if network_type == constants.TYPE_VLAN: + other_project_ranges = other_project_ranges.filter( + cls.db_model.physical_network == + _filters['physical_network']) + + for other_project_range in other_project_ranges.all(): + _set = set(six_range(other_project_range.minimum, + other_project_range.maximum + 1)) + segment_ids.difference_update(_set) + + # NOTE(ralonsoh): https://stackoverflow.com/questions/4628333/ + # converting-a-list-of-integers-into-range-in-python + segment_ranges = [ + [t[0][1], t[-1][1]] for t in + (tuple(g[1]) for g in itertools.groupby( + enumerate(segment_ids), + key=lambda enum_seg: enum_seg[1] - enum_seg[0]))] + + # Retrieve all segments belonging to the default range except those + # assigned to other projects. + query = cls._build_query_segments(context, model, network_type, + **_filters) + clauses = [and_(model_segmentation_id >= range[0], + model_segmentation_id <= range[1]) + for range in segment_ranges] + query = query.filter(or_(*clauses)) + return query.limit(common_constants.IDPOOL_SELECT_SIZE).all() diff --git a/neutron/objects/plugins/ml2/base.py b/neutron/objects/plugins/ml2/base.py index 3a4d6088300..e173b909445 100644 --- a/neutron/objects/plugins/ml2/base.py +++ b/neutron/objects/plugins/ml2/base.py @@ -12,8 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -import netaddr +import abc +import netaddr +import six + +from neutron.common import _constants as common_constants from neutron.objects import base @@ -34,3 +38,41 @@ class EndpointBase(base.NeutronDbObject): if 'ip_address' in fields: result['ip_address'] = cls.filter_to_str(result['ip_address']) return result + + +@six.add_metaclass(abc.ABCMeta) +class SegmentAllocation(object): + + @classmethod + def get_unallocated_segments(cls, context, **filters): + with cls.db_context_reader(context): + columns = set(dict(cls.db_model.__table__.columns)) + model_filters = dict((k, filters[k]) + for k in columns & set(filters.keys())) + query = context.session.query(cls.db_model).filter_by( + allocated=False, **model_filters) + return query.limit(common_constants.IDPOOL_SELECT_SIZE).all() + + @classmethod + def allocate(cls, context, **segment): + with cls.db_context_writer(context): + return context.session.query(cls.db_model).filter_by( + allocated=False, **segment).update({'allocated': True}) + + @classmethod + def deallocate(cls, context, **segment): + with cls.db_context_writer(context): + return context.session.query(cls.db_model).filter_by( + allocated=True, **segment).update({'allocated': False}) + + @classmethod + def update_primary_keys(cls, _dict, segmentation_id=None, **kwargs): + _dict[cls.primary_keys[0]] = segmentation_id + + @abc.abstractmethod + def get_segmentation_id(self): + pass + + @property + def segmentation_id(self): + return self.db_obj.segmentation_id diff --git a/neutron/objects/plugins/ml2/flatallocation.py b/neutron/objects/plugins/ml2/flatallocation.py index 88db04d0a9b..914220ebc26 100644 --- a/neutron/objects/plugins/ml2/flatallocation.py +++ b/neutron/objects/plugins/ml2/flatallocation.py @@ -10,7 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. - +from neutron_lib import constants as n_const from oslo_versionedobjects import fields as obj_fields from neutron.db.models.plugins.ml2 import flatallocation @@ -29,3 +29,5 @@ class FlatAllocation(base.NeutronDbObject): } primary_keys = ['physical_network'] + + network_type = n_const.TYPE_FLAT diff --git a/neutron/objects/plugins/ml2/geneveallocation.py b/neutron/objects/plugins/ml2/geneveallocation.py index 99f58dc8139..229e6351f2e 100644 --- a/neutron/objects/plugins/ml2/geneveallocation.py +++ b/neutron/objects/plugins/ml2/geneveallocation.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from neutron_lib import constants as n_const from oslo_versionedobjects import fields as obj_fields from neutron.db.models.plugins.ml2 import geneveallocation @@ -20,7 +21,7 @@ from neutron.objects.plugins.ml2 import base as ml2_base @base.NeutronObjectRegistry.register -class GeneveAllocation(base.NeutronDbObject): +class GeneveAllocation(base.NeutronDbObject, ml2_base.SegmentAllocation): # Version 1.0: Initial version VERSION = '1.0' @@ -33,6 +34,12 @@ class GeneveAllocation(base.NeutronDbObject): 'allocated': obj_fields.BooleanField(default=False), } + network_type = n_const.TYPE_GENEVE + + @classmethod + def get_segmentation_id(cls): + return cls.db_model.get_segmentation_id() + @base.NeutronObjectRegistry.register class GeneveEndpoint(ml2_base.EndpointBase): diff --git a/neutron/objects/plugins/ml2/greallocation.py b/neutron/objects/plugins/ml2/greallocation.py index 87391045f54..45e93857dd5 100644 --- a/neutron/objects/plugins/ml2/greallocation.py +++ b/neutron/objects/plugins/ml2/greallocation.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from neutron_lib import constants as n_const from oslo_versionedobjects import fields as obj_fields from neutron.db.models.plugins.ml2 import gre_allocation_endpoints as gre_model @@ -20,7 +21,7 @@ from neutron.objects.plugins.ml2 import base as ml2_base @base.NeutronObjectRegistry.register -class GreAllocation(base.NeutronDbObject): +class GreAllocation(base.NeutronDbObject, ml2_base.SegmentAllocation): # Version 1.0: Initial version VERSION = '1.0' @@ -33,6 +34,12 @@ class GreAllocation(base.NeutronDbObject): 'allocated': obj_fields.BooleanField(default=False) } + network_type = n_const.TYPE_GRE + + @classmethod + def get_segmentation_id(cls): + return cls.db_model.get_segmentation_id() + @base.NeutronObjectRegistry.register class GreEndpoint(ml2_base.EndpointBase): diff --git a/neutron/objects/plugins/ml2/vlanallocation.py b/neutron/objects/plugins/ml2/vlanallocation.py index 16944bc9591..3e81236f126 100644 --- a/neutron/objects/plugins/ml2/vlanallocation.py +++ b/neutron/objects/plugins/ml2/vlanallocation.py @@ -12,15 +12,17 @@ # License for the specific language governing permissions and limitations # under the License. +from neutron_lib import constants as n_const from oslo_versionedobjects import fields as obj_fields from neutron.db.models.plugins.ml2 import vlanallocation as vlan_alloc_model from neutron.objects import base from neutron.objects import common_types +from neutron.objects.plugins.ml2 import base as ml2_base @base.NeutronObjectRegistry.register -class VlanAllocation(base.NeutronDbObject): +class VlanAllocation(base.NeutronDbObject, ml2_base.SegmentAllocation): # Version 1.0: Initial version VERSION = '1.0' @@ -34,6 +36,8 @@ class VlanAllocation(base.NeutronDbObject): primary_keys = ['physical_network', 'vlan_id'] + network_type = n_const.TYPE_VLAN + @staticmethod def get_physical_networks(context): query = context.session.query(VlanAllocation.db_model.physical_network) @@ -54,3 +58,13 @@ class VlanAllocation(base.NeutronDbObject): [{'physical_network': physical_network, 'allocated': False, 'vlan_id': vlan_id} for vlan_id in vlan_ids]) + + @classmethod + def update_primary_keys(cls, _dict, segmentation_id=None, + physical_network=None): + _dict['physical_network'] = physical_network + _dict['vlan_id'] = segmentation_id + + @classmethod + def get_segmentation_id(cls): + return cls.db_model.get_segmentation_id() diff --git a/neutron/objects/plugins/ml2/vxlanallocation.py b/neutron/objects/plugins/ml2/vxlanallocation.py index 15a44123748..8de51e20900 100644 --- a/neutron/objects/plugins/ml2/vxlanallocation.py +++ b/neutron/objects/plugins/ml2/vxlanallocation.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from neutron_lib import constants as n_const from oslo_versionedobjects import fields as obj_fields from neutron.db.models.plugins.ml2 import vxlanallocation as vxlan_model @@ -21,7 +22,7 @@ from neutron.objects.plugins.ml2 import base as ml2_base @base.NeutronObjectRegistry.register -class VxlanAllocation(base.NeutronDbObject): +class VxlanAllocation(base.NeutronDbObject, ml2_base.SegmentAllocation): # Version 1.0: Initial version VERSION = '1.0' @@ -34,6 +35,12 @@ class VxlanAllocation(base.NeutronDbObject): 'allocated': obj_fields.BooleanField(default=False), } + network_type = n_const.TYPE_VXLAN + + @classmethod + def get_segmentation_id(cls): + return cls.db_model.get_segmentation_id() + @base.NeutronObjectRegistry.register class VxlanEndpoint(ml2_base.EndpointBase): diff --git a/neutron/plugins/ml2/drivers/helpers.py b/neutron/plugins/ml2/drivers/helpers.py index 1a62abeee81..9e3d97c861a 100644 --- a/neutron/plugins/ml2/drivers/helpers.py +++ b/neutron/plugins/ml2/drivers/helpers.py @@ -13,10 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. -import random +import functools -from neutron_lib import constants as p_const -from neutron_lib import context as neutron_ctx from neutron_lib.db import api as db_api from neutron_lib import exceptions from neutron_lib.plugins import constants as plugin_constants @@ -27,17 +25,12 @@ from neutron_lib.utils import helpers from oslo_config import cfg from oslo_db import exception as db_exc from oslo_log import log -from sqlalchemy import and_ -from sqlalchemy import sql -from neutron.db.models import network_segment_range as range_model -from neutron.objects import base as base_obj +from neutron.objects import network_segment_range as ns_range LOG = log.getLogger(__name__) -IDPOOL_SELECT_SIZE = 100 - class BaseTypeDriver(api.ML2TypeDriver): """BaseTypeDriver for functions common to Segment and flat.""" @@ -64,61 +57,10 @@ class SegmentTypeDriver(BaseTypeDriver): def __init__(self, model): super(SegmentTypeDriver, self).__init__() - if issubclass(model, base_obj.NeutronDbObject): - self.model = model.db_model - else: - self.model = model - self.primary_keys = set(dict(self.model.__table__.columns)) - self.primary_keys.remove("allocated") - - # TODO(ataraday): get rid of this method when old TypeDriver won't be used - def _get_session(self, arg): - if isinstance(arg, neutron_ctx.Context): - return arg.session, db_api.CONTEXT_WRITER.using(arg) - return arg, arg.session.begin(subtransactions=True) - - def build_segment_query(self, session, **filters): - # Only uses filters that correspond to columns defined by this model. - # Subclasses may use/support additional filters - columns = set(dict(self.model.__table__.columns)) - model_filters = dict((k, filters[k]) - for k in columns & set(filters.keys())) - return [session.query(self.model).filter_by(allocated=False, - **model_filters)] - - def build_segment_queries_for_tenant_and_shared_ranges(self, session, - **filters): - """Enforces that segments are allocated from network segment ranges - that are owned by the tenant, and then from shared ranges, but never - from ranges owned by other tenants. - This method also enforces that other network segment range attributes - are used when constraining the set of possible segments to be used. - """ - network_type = self.get_type() - project_id = filters.pop('project_id', None) - columns = set(dict(self.model.__table__.columns)) - model_filters = dict((k, filters[k]) - for k in columns & set(filters.keys())) - query = (session.query(self.model) - .filter_by(allocated=False, **model_filters)) - query = query.join( - range_model.NetworkSegmentRange, - and_(range_model.NetworkSegmentRange.network_type == network_type, - self.model.physical_network == - range_model.NetworkSegmentRange.physical_network if - network_type == p_const.TYPE_VLAN else - sql.expression.true())) - query = query.filter(and_(self.model_segmentation_id >= - range_model.NetworkSegmentRange.minimum, - self.model_segmentation_id <= - range_model.NetworkSegmentRange.maximum)) - query_project_id = (query.filter( - range_model.NetworkSegmentRange.project_id == project_id) if - project_id is not None else []) - query_shared = query.filter( - range_model.NetworkSegmentRange.shared == sql.expression.true()) - - return [query_project_id] + [query_shared] + self.model = model.db_model + self.segmentation_obj = model + primary_keys_columns = self.model.__table__.primary_key.columns + self.primary_keys = {col.name for col in primary_keys_columns} def allocate_fully_specified_segment(self, context, **raw_segment): """Allocate segment fully specified by raw_segment. @@ -129,12 +71,10 @@ class SegmentTypeDriver(BaseTypeDriver): """ network_type = self.get_type() - session, ctx_manager = self._get_session(context) - try: - with ctx_manager: + with db_api.CONTEXT_WRITER.using(context): alloc = ( - session.query(self.model).filter_by(**raw_segment). + context.session.query(self.model).filter_by(**raw_segment). first()) if alloc: if alloc.allocated: @@ -146,7 +86,7 @@ class SegmentTypeDriver(BaseTypeDriver): "started ", {"type": network_type, "segment": raw_segment}) - count = (session.query(self.model). + count = (context.session.query(self.model). filter_by(allocated=False, **raw_segment). update({"allocated": True})) if count: @@ -167,7 +107,7 @@ class SegmentTypeDriver(BaseTypeDriver): LOG.debug("%(type)s segment %(segment)s create started", {"type": network_type, "segment": raw_segment}) alloc = self.model(allocated=True, **raw_segment) - alloc.save(session) + alloc.save(context.session) LOG.debug("%(type)s segment %(segment)s create done", {"type": network_type, "segment": raw_segment}) @@ -184,46 +124,30 @@ class SegmentTypeDriver(BaseTypeDriver): Return allocated db object or None. """ - network_type = self.get_type() - session, ctx_manager = self._get_session(context) - with ctx_manager: - queries = (self.build_segment_queries_for_tenant_and_shared_ranges( - session, **filters) - if directory.get_plugin( - plugin_constants.NETWORK_SEGMENT_RANGE) else - self.build_segment_query(session, **filters)) + if directory.get_plugin(plugin_constants.NETWORK_SEGMENT_RANGE): + calls = [ + functools.partial( + ns_range.NetworkSegmentRange.get_segments_for_project, + context, self.model, network_type, + self.model_segmentation_id, **filters), + functools.partial( + ns_range.NetworkSegmentRange.get_segments_shared, + context, self.model, network_type, + self.model_segmentation_id, **filters)] + else: + calls = [functools.partial( + self.segmentation_obj.get_unallocated_segments, + context, **filters)] - for select in queries: - # Selected segment can be allocated before update by someone - # else - allocs = select.limit(IDPOOL_SELECT_SIZE).all() - - if not allocs: - # No resource available - continue - - alloc = random.choice(allocs) - raw_segment = dict((k, alloc[k]) for k in self.primary_keys) - LOG.debug("%(type)s segment allocate from pool " - "started with %(segment)s ", - {"type": network_type, - "segment": raw_segment}) - count = (session.query(self.model). - filter_by(allocated=False, **raw_segment). - update({"allocated": True})) - if count: - LOG.debug("%(type)s segment allocate from pool " - "success with %(segment)s ", - {"type": network_type, - "segment": raw_segment}) + for call in calls: + allocations = call() + for alloc in allocations: + segment = dict((k, alloc[k]) for k in self.primary_keys) + if self.segmentation_obj.allocate(context, **segment): + LOG.debug('%(type)s segment allocate from pool success ' + 'with %(segment)s ', {'type': network_type, + 'segment': segment}) return alloc - - # Segment allocated since select - LOG.debug("Allocate %(type)s segment from pool " - "failed with segment %(segment)s", - {"type": network_type, - "segment": raw_segment}) - # saving real exception in case we exceeded amount of attempts raise db_exc.RetryRequest( exceptions.NoNetworkFoundInMaximumAllowedAttempts()) diff --git a/neutron/tests/unit/objects/plugins/ml2/test_base.py b/neutron/tests/unit/objects/plugins/ml2/test_base.py new file mode 100644 index 00000000000..d7993d62d58 --- /dev/null +++ b/neutron/tests/unit/objects/plugins/ml2/test_base.py @@ -0,0 +1,34 @@ +# Copyright (c) 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class SegmentAllocationDbObjTestCase(object): + + def test_get_unallocated_segments(self): + self.assertEqual( + [], self._test_class.get_unallocated_segments(self.context)) + + obj = self.objs[0] + obj.allocated = True + obj.create() + self.assertEqual( + [], self._test_class.get_unallocated_segments(self.context)) + + obj = self.objs[1] + obj.allocated = False + obj.create() + allocations = self._test_class.get_unallocated_segments(self.context) + self.assertEqual(1, len(allocations)) + self.assertEqual(obj.segmentation_id, allocations[0].segmentation_id) diff --git a/neutron/tests/unit/objects/plugins/ml2/test_geneveallocation.py b/neutron/tests/unit/objects/plugins/ml2/test_geneveallocation.py index 2ca836c2824..931cc943cca 100644 --- a/neutron/tests/unit/objects/plugins/ml2/test_geneveallocation.py +++ b/neutron/tests/unit/objects/plugins/ml2/test_geneveallocation.py @@ -13,6 +13,7 @@ # under the License. from neutron.objects.plugins.ml2 import geneveallocation +from neutron.tests.unit.objects.plugins.ml2 import test_base as ml2_test_base from neutron.tests.unit.objects import test_base from neutron.tests.unit import testlib_api @@ -22,8 +23,9 @@ class GeneveAllocationIfaceObjTestCase(test_base.BaseObjectIfaceTestCase): _test_class = geneveallocation.GeneveAllocation -class GeneveAllocationDbObjTestCase(test_base.BaseDbObjectTestCase, - testlib_api.SqlTestCase): +class GeneveAllocationDbObjTestCase( + test_base.BaseDbObjectTestCase, testlib_api.SqlTestCase, + ml2_test_base.SegmentAllocationDbObjTestCase): _test_class = geneveallocation.GeneveAllocation diff --git a/neutron/tests/unit/objects/plugins/ml2/test_greallocation.py b/neutron/tests/unit/objects/plugins/ml2/test_greallocation.py index d56ac5f467f..6512d56f802 100644 --- a/neutron/tests/unit/objects/plugins/ml2/test_greallocation.py +++ b/neutron/tests/unit/objects/plugins/ml2/test_greallocation.py @@ -13,6 +13,7 @@ # under the License. from neutron.objects.plugins.ml2 import greallocation as gre_object +from neutron.tests.unit.objects.plugins.ml2 import test_base as ml2_test_base from neutron.tests.unit.objects import test_base from neutron.tests.unit import testlib_api @@ -22,8 +23,9 @@ class GreAllocationIfaceObjTestCase(test_base.BaseObjectIfaceTestCase): _test_class = gre_object.GreAllocation -class GreAllocationDbObjTestCase(test_base.BaseDbObjectTestCase, - testlib_api.SqlTestCase): +class GreAllocationDbObjTestCase( + test_base.BaseDbObjectTestCase, testlib_api.SqlTestCase, + ml2_test_base.SegmentAllocationDbObjTestCase): _test_class = gre_object.GreAllocation diff --git a/neutron/tests/unit/objects/plugins/ml2/test_vlanallocation.py b/neutron/tests/unit/objects/plugins/ml2/test_vlanallocation.py index 7a775c866d8..4d48819db6d 100644 --- a/neutron/tests/unit/objects/plugins/ml2/test_vlanallocation.py +++ b/neutron/tests/unit/objects/plugins/ml2/test_vlanallocation.py @@ -13,6 +13,7 @@ # under the License. from neutron.objects.plugins.ml2 import vlanallocation +from neutron.tests.unit.objects.plugins.ml2 import test_base as ml2_test_base from neutron.tests.unit.objects import test_base from neutron.tests.unit import testlib_api @@ -22,7 +23,8 @@ class VlanAllocationIfaceObjTestCase(test_base.BaseObjectIfaceTestCase): _test_class = vlanallocation.VlanAllocation -class VlanAllocationDbObjTestCase(test_base.BaseDbObjectTestCase, - testlib_api.SqlTestCase): +class VlanAllocationDbObjTestCase( + test_base.BaseDbObjectTestCase, testlib_api.SqlTestCase, + ml2_test_base.SegmentAllocationDbObjTestCase): _test_class = vlanallocation.VlanAllocation diff --git a/neutron/tests/unit/objects/plugins/ml2/test_vxlanallocation.py b/neutron/tests/unit/objects/plugins/ml2/test_vxlanallocation.py index ceb343139a6..a531c0c2b65 100644 --- a/neutron/tests/unit/objects/plugins/ml2/test_vxlanallocation.py +++ b/neutron/tests/unit/objects/plugins/ml2/test_vxlanallocation.py @@ -13,6 +13,7 @@ # under the License. from neutron.objects.plugins.ml2 import vxlanallocation as vxlan_obj +from neutron.tests.unit.objects.plugins.ml2 import test_base as ml2_test_base from neutron.tests.unit.objects import test_base from neutron.tests.unit import testlib_api @@ -22,8 +23,9 @@ class VxlanAllocationIfaceObjTestCase(test_base.BaseObjectIfaceTestCase): _test_class = vxlan_obj.VxlanAllocation -class VxlanAllocationDbObjTestCase(test_base.BaseDbObjectTestCase, - testlib_api.SqlTestCase): +class VxlanAllocationDbObjTestCase( + test_base.BaseDbObjectTestCase, testlib_api.SqlTestCase, + ml2_test_base.SegmentAllocationDbObjTestCase): _test_class = vxlan_obj.VxlanAllocation diff --git a/neutron/tests/unit/objects/test_network_segment_range.py b/neutron/tests/unit/objects/test_network_segment_range.py index 41f23fefe64..e076e896ffe 100644 --- a/neutron/tests/unit/objects/test_network_segment_range.py +++ b/neutron/tests/unit/objects/test_network_segment_range.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import itertools import random import mock @@ -22,12 +23,14 @@ from oslo_utils import uuidutils from neutron.objects import network as net_obj from neutron.objects import network_segment_range +from neutron.objects.plugins.ml2 import base as ml2_base from neutron.objects.plugins.ml2 import vlanallocation as vlan_alloc_obj from neutron.tests.unit.objects import test_base as obj_test_base from neutron.tests.unit import testlib_api TEST_TENANT_ID = '46f70361-ba71-4bd0-9769-3573fd227c4b' TEST_PHYSICAL_NETWORK = 'phys_net' +NUM_ALLOCATIONS = 3 class NetworkSegmentRangeIfaceObjectTestCase( @@ -68,16 +71,16 @@ class NetworkSegmentRangeDbObjectTestCase(obj_test_base.BaseDbObjectTestCase, _test_class = network_segment_range.NetworkSegmentRange - def _create_test_vlan_allocation(self, vlan_id=None, allocated=False): - attr = self.get_random_object_fields(vlan_alloc_obj.VlanAllocation) - attr.update({ - 'vlan_id': vlan_id or random.randint( - constants.MIN_VLAN_TAG, constants.MAX_VLAN_TAG), - 'physical_network': 'foo', - 'allocated': allocated}) - _vlan_allocation = vlan_alloc_obj.VlanAllocation(self.context, **attr) - _vlan_allocation.create() - return _vlan_allocation + def _create_allocation(self, allocation_class, segmentation_id=None, + physical_network=None, allocated=False): + attr = self.get_random_object_fields(allocation_class) + attr['allocated'] = allocated + allocation_class.update_primary_keys( + attr, segmentation_id=segmentation_id, + physical_network=physical_network or 'foo') + allocation = allocation_class(self.context, **attr) + allocation.create() + return allocation def _create_test_network(self, name=None, network_id=None): name = "test-network-%s" % helpers.get_random_string(4) @@ -88,24 +91,30 @@ class NetworkSegmentRangeDbObjectTestCase(obj_test_base.BaseDbObjectTestCase, _network.create() return _network - def _create_test_vlan_segment(self, segmentation_id=None, network_id=None): + def _create_segment(self, segmentation_id=None, network_id=None, + physical_network=None, network_type=None): attr = self.get_random_object_fields(net_obj.NetworkSegment) attr.update({ 'network_id': network_id or self._create_test_network_id(), - 'network_type': constants.TYPE_VLAN, - 'physical_network': 'foo', + 'network_type': network_type or constants.TYPE_VLAN, + 'physical_network': physical_network or 'foo', 'segmentation_id': segmentation_id or random.randint( constants.MIN_VLAN_TAG, constants.MAX_VLAN_TAG)}) _segment = net_obj.NetworkSegment(self.context, **attr) _segment.create() return _segment - def _create_test_vlan_network_segment_range_obj(self, minimum, maximum): + def _create_network_segment_range( + self, minimum, maximum, network_type=None, physical_network=None, + project_id=None, default=False): kwargs = self.get_random_db_fields() - kwargs.update({'network_type': constants.TYPE_VLAN, - 'physical_network': 'foo', + kwargs.update({'network_type': network_type or constants.TYPE_VLAN, + 'physical_network': physical_network or 'foo', 'minimum': minimum, - 'maximum': maximum}) + 'maximum': maximum, + 'default': default, + 'shared': default, + 'project_id': project_id}) db_obj = self._test_class.db_model(**kwargs) obj_fields = self._test_class.modify_fields_from_db(db_obj) obj = self._test_class(self.context, **obj_fields) @@ -117,11 +126,14 @@ class NetworkSegmentRangeDbObjectTestCase(obj_test_base.BaseDbObjectTestCase, to_alloc = range(range_minimum, range_maximum - 5) not_to_alloc = range(range_maximum - 5, range_maximum + 1) for vlan_id in to_alloc: - self._create_test_vlan_allocation(vlan_id=vlan_id, allocated=True) + self._create_allocation(vlan_alloc_obj.VlanAllocation, + segmentation_id=vlan_id, allocated=True, + physical_network='foo') for vlan_id in not_to_alloc: - self._create_test_vlan_allocation(vlan_id=vlan_id, allocated=False) - obj = self._create_test_vlan_network_segment_range_obj(range_minimum, - range_maximum) + self._create_allocation(vlan_alloc_obj.VlanAllocation, + segmentation_id=vlan_id, allocated=False, + physical_network='foo') + obj = self._create_network_segment_range(range_minimum, range_maximum) available_alloc = self._test_class._get_available_allocation(obj) self.assertItemsEqual(not_to_alloc, available_alloc) @@ -129,10 +141,10 @@ class NetworkSegmentRangeDbObjectTestCase(obj_test_base.BaseDbObjectTestCase, alloc_mapping = {} for _ in range(5): network = self._create_test_network() - segment = self._create_test_vlan_segment(network_id=network.id) + segment = self._create_segment(network_id=network.id) alloc_mapping.update({segment.segmentation_id: network.project_id}) - obj = self._create_test_vlan_network_segment_range_obj( + obj = self._create_network_segment_range( minimum=min(list(alloc_mapping.keys())), maximum=max(list(alloc_mapping.keys()))) ret_alloc_mapping = self._test_class._get_used_allocation_mapping(obj) @@ -166,3 +178,99 @@ class NetworkSegmentRangeDbObjectTestCase(obj_test_base.BaseDbObjectTestCase, obj.create() obj.shared = False self.assertRaises(n_exc.ObjectActionError, obj.update) + + def _create_environment(self): + self.projects = [uuidutils.generate_uuid() for _ in range(3)] + self.segment_ranges = { + 'default': [100, 120], self.projects[0]: [90, 105], + self.projects[1]: [109, 114], self.projects[2]: [117, 130]} + self.seg_min = self.segment_ranges['default'][0] + self.seg_max = self.segment_ranges['default'][1] + + for subclass in ml2_base.SegmentAllocation.__subclasses__(): + # Build segment ranges: default one and project specific ones. + for name, ranges in self.segment_ranges.items(): + default = True if name == 'default' else False + project = name if not default else None + self._create_network_segment_range( + ranges[0], ranges[1], network_type=subclass.network_type, + project_id=project, default=default).create() + + # Build allocations (non allocated). + for segmentation_id in range(self.seg_min, self.seg_max + 1): + self._create_allocation(subclass, + segmentation_id=segmentation_id) + + def _default_range_set(self, project_id=None): + range_set = set(range(self.segment_ranges['default'][0], + self.segment_ranges['default'][1] + 1)) + for p_id, ranges in ((p, r) for (p, r) in self.segment_ranges.items() + if p not in [project_id, 'default']): + pranges = self.segment_ranges.get(p_id, [0, 0]) + prange_set = set(range(pranges[0], pranges[1] + 1)) + range_set.difference_update(prange_set) + return range_set + + def _allocate_random_allocations(self, allocations, subclass): + pk_cols = subclass.db_model.__table__.primary_key.columns + primary_keys = [col.name for col in pk_cols] + allocated = [] + for allocation in random.sample(allocations, k=NUM_ALLOCATIONS): + segment = dict((k, allocation[k]) for k in primary_keys) + allocated.append(segment) + self.assertEqual(1, subclass.allocate(self.context, **segment)) + return allocated + + def test_get_segments_for_project(self): + self._create_environment() + for project_id, subclass in itertools.product( + self.projects, ml2_base.SegmentAllocation.__subclasses__()): + allocations = network_segment_range.NetworkSegmentRange. \ + get_segments_for_project( + self.context, subclass.db_model, subclass.network_type, + subclass.get_segmentation_id(), project_id=project_id) + project_min = max(self.seg_min, self.segment_ranges[project_id][0]) + project_max = min(self.seg_max, self.segment_ranges[project_id][1]) + project_segment_ids = list(range(project_min, project_max + 1)) + self.assertEqual(len(allocations), len(project_segment_ids)) + for allocation in allocations: + self.assertFalse(allocation.allocated) + self.assertIn(allocation.segmentation_id, project_segment_ids) + + # Allocate random segments inside the project range. + self._allocate_random_allocations(allocations, subclass) + allocations = network_segment_range.NetworkSegmentRange. \ + get_segments_for_project( + self.context, subclass.db_model, subclass.network_type, + subclass.get_segmentation_id(), project_id=project_id) + self.assertEqual(len(allocations), + len(project_segment_ids) - NUM_ALLOCATIONS) + + def test_get_segments_shared(self): + self._create_environment() + self.projects.append(None) + for project_id, subclass in itertools.product( + self.projects, ml2_base.SegmentAllocation.__subclasses__()): + filters = {'project_id': project_id, + 'physical_network': 'foo'} + allocations = network_segment_range.NetworkSegmentRange. \ + get_segments_shared( + self.context, subclass.db_model, subclass.network_type, + subclass.get_segmentation_id(), **filters) + + prange = self._default_range_set(project_id) + self.assertEqual(len(prange), len(allocations)) + + # Allocate random segments inside the project shared range. + allocated = self._allocate_random_allocations(allocations, + subclass) + allocations = network_segment_range.NetworkSegmentRange. \ + get_segments_shared( + self.context, subclass.db_model, subclass.network_type, + subclass.get_segmentation_id(), **filters) + self.assertEqual(len(allocations), len(prange) - NUM_ALLOCATIONS) + + # Deallocate the allocated segments because can be allocated in + # a segmentation ID not belonging to any project. + for alloc in allocated: + self.assertEqual(1, subclass.deallocate(self.context, **alloc))