From a01b7125cd965625316d9aec3a7408612b94fc08 Mon Sep 17 00:00:00 2001 From: Kailun Qin Date: Tue, 5 Mar 2019 18:47:08 +0800 Subject: [PATCH] Use network segment ranges for segment allocation This patch makes necessary changes to ML2 type drivers and plugin manager for network segment range extension support when it is loaded. When the network segment range extension is not loaded, no impact to the current flow. When the extension is loaded, - populating a range that is managed from the configuration file [1]_, such as "VLAN IDs", "VXLAN VNI IDs", "GRE tunnel IDs", "Geneve VNI IDs" to the network segment range DB table as a "default" and "shared" entry to maintain backward compatibility; - reloading the "default" segment ranges when Neutron server starts/restarts; - creating a set of "default" network segment ranges out of the ML2-config-file-defined ranges [1]_ and the segment allocation operations are always retrieving the information from the DB to have the network segment ranges fully administered via API; - when a tenant allocates a segment, it will first allocate from an available segment range assigned to the tenant, and then a shared range if no tenant specific allocation is possible. [1] /etc/neutron/plugins/ml2/ml2_conf.ini Co-authored-by: Allain Legacy Partially-implements: blueprint network-segment-range-management Change-Id: I522940fc4d054f5eec1110eb2c424e32e8ae6bad --- neutron/plugins/ml2/drivers/helpers.py | 108 ++++++++++++---- neutron/plugins/ml2/drivers/type_flat.py | 13 +- neutron/plugins/ml2/drivers/type_geneve.py | 4 + neutron/plugins/ml2/drivers/type_gre.py | 3 + neutron/plugins/ml2/drivers/type_local.py | 13 +- neutron/plugins/ml2/drivers/type_tunnel.py | 115 +++++++++++++++-- neutron/plugins/ml2/drivers/type_vlan.py | 118 ++++++++++++++++-- neutron/plugins/ml2/drivers/type_vxlan.py | 3 + neutron/plugins/ml2/managers.py | 43 +++++-- .../services/network_segment_range/plugin.py | 8 ++ .../plugins/ml2/drivers/base_type_tunnel.py | 43 +++++++ .../unit/plugins/ml2/drivers/test_helpers.py | 22 ++++ .../plugins/ml2/drivers/test_type_geneve.py | 1 + .../unit/plugins/ml2/drivers/test_type_gre.py | 1 + .../plugins/ml2/drivers/test_type_vlan.py | 41 ++++++ .../plugins/ml2/drivers/test_type_vxlan.py | 1 + 16 files changed, 472 insertions(+), 65 deletions(-) diff --git a/neutron/plugins/ml2/drivers/helpers.py b/neutron/plugins/ml2/drivers/helpers.py index ac92510f6b3..866ecf1b5ff 100644 --- a/neutron/plugins/ml2/drivers/helpers.py +++ b/neutron/plugins/ml2/drivers/helpers.py @@ -15,16 +15,22 @@ import random +from neutron_lib import constants as p_const from neutron_lib import context as neutron_ctx from neutron_lib.db import api as db_api from neutron_lib import exceptions +from neutron_lib.plugins import constants as plugin_constants +from neutron_lib.plugins import directory from neutron_lib.plugins.ml2 import api from neutron_lib.plugins import utils as p_utils from neutron_lib.utils import helpers from oslo_config import cfg from oslo_db import exception as db_exc from oslo_log import log +from sqlalchemy import and_ +from sqlalchemy import sql +from neutron.db.models import network_segment_range as range_model from neutron.objects import base as base_obj @@ -71,6 +77,49 @@ class SegmentTypeDriver(BaseTypeDriver): return arg.session, db_api.CONTEXT_WRITER.using(arg) return arg, arg.session.begin(subtransactions=True) + def build_segment_query(self, session, **filters): + # Only uses filters that correspond to columns defined by this model. + # Subclasses may use/support additional filters + columns = set(dict(self.model.__table__.columns)) + model_filters = dict((k, filters[k]) + for k in columns & set(filters.keys())) + return [session.query(self.model).filter_by(allocated=False, + **model_filters)] + + def build_segment_queries_for_tenant_and_shared_ranges(self, session, + **filters): + """Enforces that segments are allocated from network segment ranges + that are owned by the tenant, and then from shared ranges, but never + from ranges owned by other tenants. + This method also enforces that other network segment range attributes + are used when constraining the set of possible segments to be used. + """ + network_type = self.get_type() + project_id = filters.pop('project_id', None) + columns = set(dict(self.model.__table__.columns)) + model_filters = dict((k, filters[k]) + for k in columns & set(filters.keys())) + query = (session.query(self.model) + .filter_by(allocated=False, **model_filters)) + query = query.join( + range_model.NetworkSegmentRange, + and_(range_model.NetworkSegmentRange.network_type == network_type, + self.model.physical_network == + range_model.NetworkSegmentRange.physical_network if + network_type == p_const.TYPE_VLAN else + sql.expression.true())) + query = query.filter(and_(self.model_segmentation_id >= + range_model.NetworkSegmentRange.minimum, + self.model_segmentation_id <= + range_model.NetworkSegmentRange.maximum)) + query_project_id = (query.filter( + range_model.NetworkSegmentRange.project_id == project_id) if + project_id is not None else []) + query_shared = query.filter( + range_model.NetworkSegmentRange.shared == sql.expression.true()) + + return [query_project_id] + [query_shared] + def allocate_fully_specified_segment(self, context, **raw_segment): """Allocate segment fully specified by raw_segment. @@ -139,37 +188,42 @@ class SegmentTypeDriver(BaseTypeDriver): network_type = self.get_type() session, ctx_manager = self._get_session(context) with ctx_manager: - select = (session.query(self.model). - filter_by(allocated=False, **filters)) + queries = (self.build_segment_queries_for_tenant_and_shared_ranges( + session, **filters) + if directory.get_plugin( + plugin_constants.NETWORK_SEGMENT_RANGE) else + self.build_segment_query(session, **filters)) - # Selected segment can be allocated before update by someone else, - allocs = select.limit(IDPOOL_SELECT_SIZE).all() + for select in queries: + # Selected segment can be allocated before update by someone + # else + allocs = select.limit(IDPOOL_SELECT_SIZE).all() - if not allocs: - # No resource available - return + if not allocs: + # No resource available + continue - alloc = random.choice(allocs) - raw_segment = dict((k, alloc[k]) for k in self.primary_keys) - LOG.debug("%(type)s segment allocate from pool " - "started with %(segment)s ", - {"type": network_type, - "segment": raw_segment}) - count = (session.query(self.model). - filter_by(allocated=False, **raw_segment). - update({"allocated": True})) - if count: + alloc = random.choice(allocs) + raw_segment = dict((k, alloc[k]) for k in self.primary_keys) LOG.debug("%(type)s segment allocate from pool " - "success with %(segment)s ", + "started with %(segment)s ", {"type": network_type, "segment": raw_segment}) - return alloc + count = (session.query(self.model). + filter_by(allocated=False, **raw_segment). + update({"allocated": True})) + if count: + LOG.debug("%(type)s segment allocate from pool " + "success with %(segment)s ", + {"type": network_type, + "segment": raw_segment}) + return alloc - # Segment allocated since select - LOG.debug("Allocate %(type)s segment from pool " - "failed with segment %(segment)s", - {"type": network_type, - "segment": raw_segment}) - # saving real exception in case we exceeded amount of attempts - raise db_exc.RetryRequest( - exceptions.NoNetworkFoundInMaximumAllowedAttempts()) + # Segment allocated since select + LOG.debug("Allocate %(type)s segment from pool " + "failed with segment %(segment)s", + {"type": network_type, + "segment": raw_segment}) + # saving real exception in case we exceeded amount of attempts + raise db_exc.RetryRequest( + exceptions.NoNetworkFoundInMaximumAllowedAttempts()) diff --git a/neutron/plugins/ml2/drivers/type_flat.py b/neutron/plugins/ml2/drivers/type_flat.py index a0a0add4a58..fd11bc1b8f2 100644 --- a/neutron/plugins/ml2/drivers/type_flat.py +++ b/neutron/plugins/ml2/drivers/type_flat.py @@ -63,6 +63,15 @@ class FlatTypeDriver(helpers.BaseTypeDriver): def initialize(self): LOG.info("ML2 FlatTypeDriver initialization complete") + def initialize_network_segment_range_support(self): + pass + + def update_network_segment_range_allocations(self): + pass + + def get_network_segment_ranges(self): + pass + def is_partial_segment(self, segment): return False @@ -85,7 +94,7 @@ class FlatTypeDriver(helpers.BaseTypeDriver): msg = _("%s prohibited for flat provider network") % key raise exc.InvalidInput(error_message=msg) - def reserve_provider_segment(self, context, segment): + def reserve_provider_segment(self, context, segment, filters=None): physical_network = segment[api.PHYSICAL_NETWORK] try: LOG.debug("Reserving flat network on physical " @@ -100,7 +109,7 @@ class FlatTypeDriver(helpers.BaseTypeDriver): segment[api.MTU] = self.get_mtu(alloc.physical_network) return segment - def allocate_tenant_segment(self, context): + def allocate_tenant_segment(self, context, filters=None): # Tenant flat networks are not supported. return diff --git a/neutron/plugins/ml2/drivers/type_geneve.py b/neutron/plugins/ml2/drivers/type_geneve.py index 04e574698ec..9341ff15616 100644 --- a/neutron/plugins/ml2/drivers/type_geneve.py +++ b/neutron/plugins/ml2/drivers/type_geneve.py @@ -20,6 +20,8 @@ from oslo_config import cfg from oslo_log import log from neutron.conf.plugins.ml2.drivers import driver_type +from neutron.db.models.plugins.ml2 import geneveallocation as \ + geneve_alloc_model from neutron.objects.plugins.ml2 import geneveallocation as geneve_obj from neutron.plugins.ml2.drivers import type_tunnel @@ -34,6 +36,8 @@ class GeneveTypeDriver(type_tunnel.EndpointTunnelTypeDriver): super(GeneveTypeDriver, self).__init__(geneve_obj.GeneveAllocation, geneve_obj.GeneveEndpoint) self.max_encap_size = cfg.CONF.ml2_type_geneve.max_header_size + self.model_segmentation_id = ( + geneve_alloc_model.GeneveAllocation.geneve_vni) def get_type(self): return p_const.TYPE_GENEVE diff --git a/neutron/plugins/ml2/drivers/type_gre.py b/neutron/plugins/ml2/drivers/type_gre.py index 52edc5fd8f5..45561acd4b1 100644 --- a/neutron/plugins/ml2/drivers/type_gre.py +++ b/neutron/plugins/ml2/drivers/type_gre.py @@ -19,6 +19,8 @@ from oslo_config import cfg from oslo_log import log from neutron.conf.plugins.ml2.drivers import driver_type +from neutron.db.models.plugins.ml2 import gre_allocation_endpoints as \ + gre_alloc_model from neutron.objects.plugins.ml2 import greallocation as gre_obj from neutron.plugins.ml2.drivers import type_tunnel @@ -32,6 +34,7 @@ class GreTypeDriver(type_tunnel.EndpointTunnelTypeDriver): def __init__(self): super(GreTypeDriver, self).__init__( gre_obj.GreAllocation, gre_obj.GreEndpoint) + self.model_segmentation_id = gre_alloc_model.GreAllocation.gre_id def get_type(self): return p_const.TYPE_GRE diff --git a/neutron/plugins/ml2/drivers/type_local.py b/neutron/plugins/ml2/drivers/type_local.py index 61a005282cb..be223a8f2d7 100644 --- a/neutron/plugins/ml2/drivers/type_local.py +++ b/neutron/plugins/ml2/drivers/type_local.py @@ -42,6 +42,15 @@ class LocalTypeDriver(api.ML2TypeDriver): def initialize(self): pass + def initialize_network_segment_range_support(self): + pass + + def update_network_segment_range_allocations(self): + pass + + def get_network_segment_ranges(self): + pass + def is_partial_segment(self, segment): return False @@ -51,11 +60,11 @@ class LocalTypeDriver(api.ML2TypeDriver): msg = _("%s prohibited for local provider network") % key raise exc.InvalidInput(error_message=msg) - def reserve_provider_segment(self, context, segment): + def reserve_provider_segment(self, context, segment, filters=None): # No resources to reserve return segment - def allocate_tenant_segment(self, context): + def allocate_tenant_segment(self, context, filters=None): # No resources to allocate return {api.NETWORK_TYPE: p_const.TYPE_LOCAL} diff --git a/neutron/plugins/ml2/drivers/type_tunnel.py b/neutron/plugins/ml2/drivers/type_tunnel.py index 4b674a09775..bebc44c6a7a 100644 --- a/neutron/plugins/ml2/drivers/type_tunnel.py +++ b/neutron/plugins/ml2/drivers/type_tunnel.py @@ -22,18 +22,23 @@ from neutron_lib import constants as p_const from neutron_lib import context from neutron_lib.db import api as db_api from neutron_lib import exceptions as exc +from neutron_lib.plugins import constants as plugin_constants +from neutron_lib.plugins import directory from neutron_lib.plugins.ml2 import api from neutron_lib.plugins import utils as plugin_utils from oslo_config import cfg from oslo_db import exception as db_exc from oslo_log import log +from oslo_utils import uuidutils import six from six import moves from sqlalchemy import or_ from neutron._i18n import _ from neutron.objects import base as base_obj +from neutron.objects import network_segment_range as range_obj from neutron.plugins.ml2.drivers import helpers +from neutron.services.network_segment_range import plugin as range_plugin LOG = log.getLogger(__name__) @@ -119,7 +124,15 @@ class _TunnelTypeDriverBase(helpers.SegmentTypeDriver): def _initialize(self, raw_tunnel_ranges): self.tunnel_ranges = [] self._parse_tunnel_ranges(raw_tunnel_ranges, self.tunnel_ranges) - self.sync_allocations() + if not range_plugin.is_network_segment_range_enabled(): + # service plugins are initialized/loaded after the ML2 driver + # initialization. Thus, we base on the information whether + # ``network_segment_range`` service plugin is enabled/defined in + # ``neutron.conf`` to decide whether to skip the first time sync + # allocation during driver initialization, instead of using the + # directory.get_plugin() method - the normal way used elsewhere to + # check if a plugin is loaded. + self.sync_allocations() def _parse_tunnel_ranges(self, tunnel_ranges, current_range): for entry in tunnel_ranges: @@ -136,11 +149,67 @@ class _TunnelTypeDriverBase(helpers.SegmentTypeDriver): LOG.info("%(type)s ID ranges: %(range)s", {'type': self.get_type(), 'range': current_range}) + @db_api.retry_db_errors + def _populate_new_default_network_segment_ranges(self): + ctx = context.get_admin_context() + for tun_min, tun_max in self.tunnel_ranges: + res = { + 'id': uuidutils.generate_uuid(), + 'name': '', + 'default': True, + 'shared': True, + 'network_type': self.get_type(), + 'minimum': tun_min, + 'maximum': tun_max} + with db_api.CONTEXT_WRITER.using(ctx): + new_default_range_obj = ( + range_obj.NetworkSegmentRange(ctx, **res)) + new_default_range_obj.create() + + @db_api.retry_db_errors + def _delete_expired_default_network_segment_ranges(self): + ctx = context.get_admin_context() + with db_api.CONTEXT_WRITER.using(ctx): + filters = { + 'default': True, + 'network_type': self.get_type(), + } + old_default_range_objs = range_obj.NetworkSegmentRange.get_objects( + ctx, **filters) + for obj in old_default_range_objs: + obj.delete() + + @db_api.retry_db_errors + def _get_network_segment_ranges_from_db(self): + ranges = [] + ctx = context.get_admin_context() + with db_api.CONTEXT_READER.using(ctx): + range_objs = (range_obj.NetworkSegmentRange.get_objects( + ctx, network_type=self.get_type())) + for obj in range_objs: + ranges.append((obj['minimum'], obj['maximum'])) + + return ranges + + def initialize_network_segment_range_support(self): + self._delete_expired_default_network_segment_ranges() + self._populate_new_default_network_segment_ranges() + # Override self.tunnel_ranges with the network segment range + # information from DB and then do a sync_allocations since the + # segment range service plugin has not yet been loaded at this + # initialization time. + self.tunnel_ranges = self._get_network_segment_ranges_from_db() + self.sync_allocations() + + def update_network_segment_range_allocations(self): + self.sync_allocations() + @db_api.retry_db_errors def sync_allocations(self): # determine current configured allocatable tunnel ids tunnel_ids = set() - for tun_min, tun_max in self.tunnel_ranges: + ranges = self.get_network_segment_ranges() + for tun_min, tun_max in ranges: tunnel_ids |= set(moves.range(tun_min, tun_max + 1)) tunnel_id_getter = operator.attrgetter(self.segmentation_key) @@ -198,6 +267,19 @@ class _TunnelTypeDriverBase(helpers.SegmentTypeDriver): ip_header_length = p_const.IP_HEADER_LENGTH[version] return min(mtu) - ip_header_length if mtu else 0 + def get_network_segment_ranges(self): + """Get the driver network segment ranges. + + Queries all tunnel network segment ranges from DB if the + ``NETWORK_SEGMENT_RANGE`` service plugin is enabled. Otherwise, + they will be loaded from the host config file - `ml2_conf.ini`. + """ + ranges = self.tunnel_ranges + if directory.get_plugin(plugin_constants.NETWORK_SEGMENT_RANGE): + ranges = self._get_network_segment_ranges_from_db() + + return ranges + @six.add_metaclass(abc.ABCMeta) class TunnelTypeDriver(_TunnelTypeDriverBase): @@ -213,9 +295,11 @@ class TunnelTypeDriver(_TunnelTypeDriverBase): - get_allocation """ - def reserve_provider_segment(self, session, segment): + def reserve_provider_segment(self, session, segment, filters=None): if self.is_partial_segment(segment): - alloc = self.allocate_partially_specified_segment(session) + filters = filters or {} + alloc = self.allocate_partially_specified_segment(session, + **filters) if not alloc: raise exc.NoNetworkAvailable() else: @@ -229,8 +313,9 @@ class TunnelTypeDriver(_TunnelTypeDriverBase): api.SEGMENTATION_ID: getattr(alloc, self.segmentation_key), api.MTU: self.get_mtu()} - def allocate_tenant_segment(self, session): - alloc = self.allocate_partially_specified_segment(session) + def allocate_tenant_segment(self, session, filters=None): + filters = filters or {} + alloc = self.allocate_partially_specified_segment(session, **filters) if not alloc: return return {api.NETWORK_TYPE: self.get_type(), @@ -241,7 +326,9 @@ class TunnelTypeDriver(_TunnelTypeDriverBase): def release_segment(self, session, segment): tunnel_id = segment[api.SEGMENTATION_ID] - inside = any(lo <= tunnel_id <= hi for lo, hi in self.tunnel_ranges) + ranges = self.get_network_segment_ranges() + + inside = any(lo <= tunnel_id <= hi for lo, hi in ranges) info = {'type': self.get_type(), 'id': tunnel_id} with session.begin(subtransactions=True): @@ -281,9 +368,11 @@ class ML2TunnelTypeDriver(_TunnelTypeDriverBase): - get_allocation """ - def reserve_provider_segment(self, context, segment): + def reserve_provider_segment(self, context, segment, filters=None): if self.is_partial_segment(segment): - alloc = self.allocate_partially_specified_segment(context) + filters = filters or {} + alloc = self.allocate_partially_specified_segment(context, + **filters) if not alloc: raise exc.NoNetworkAvailable() else: @@ -297,8 +386,9 @@ class ML2TunnelTypeDriver(_TunnelTypeDriverBase): api.SEGMENTATION_ID: getattr(alloc, self.segmentation_key), api.MTU: self.get_mtu()} - def allocate_tenant_segment(self, context): - alloc = self.allocate_partially_specified_segment(context) + def allocate_tenant_segment(self, context, filters=None): + filters = filters or {} + alloc = self.allocate_partially_specified_segment(context, **filters) if not alloc: return return {api.NETWORK_TYPE: self.get_type(), @@ -309,7 +399,8 @@ class ML2TunnelTypeDriver(_TunnelTypeDriverBase): def release_segment(self, context, segment): tunnel_id = segment[api.SEGMENTATION_ID] - inside = any(lo <= tunnel_id <= hi for lo, hi in self.tunnel_ranges) + ranges = self.get_network_segment_ranges() + inside = any(lo <= tunnel_id <= hi for lo, hi in ranges) info = {'type': self.get_type(), 'id': tunnel_id} with db_api.CONTEXT_WRITER.using(context): diff --git a/neutron/plugins/ml2/drivers/type_vlan.py b/neutron/plugins/ml2/drivers/type_vlan.py index ca262376e7f..2e2471e2116 100644 --- a/neutron/plugins/ml2/drivers/type_vlan.py +++ b/neutron/plugins/ml2/drivers/type_vlan.py @@ -19,16 +19,22 @@ from neutron_lib import constants as p_const from neutron_lib import context from neutron_lib.db import api as db_api from neutron_lib import exceptions as exc +from neutron_lib.plugins import constants as plugin_constants +from neutron_lib.plugins import directory from neutron_lib.plugins.ml2 import api from neutron_lib.plugins import utils as plugin_utils from oslo_config import cfg from oslo_log import log +from oslo_utils import uuidutils from six import moves from neutron._i18n import _ from neutron.conf.plugins.ml2.drivers import driver_type +from neutron.db.models.plugins.ml2 import vlanallocation as vlan_alloc_model +from neutron.objects import network_segment_range as range_obj from neutron.objects.plugins.ml2 import vlanallocation as vlanalloc from neutron.plugins.ml2.drivers import helpers +from neutron.services.network_segment_range import plugin as range_plugin LOG = log.getLogger(__name__) @@ -48,8 +54,42 @@ class VlanTypeDriver(helpers.SegmentTypeDriver): def __init__(self): super(VlanTypeDriver, self).__init__(vlanalloc.VlanAllocation) + self.model_segmentation_id = vlan_alloc_model.VlanAllocation.vlan_id self._parse_network_vlan_ranges() + @db_api.retry_db_errors + def _populate_new_default_network_segment_ranges(self): + ctx = context.get_admin_context() + for (physical_network, vlan_ranges) in ( + self.network_vlan_ranges.items()): + for vlan_min, vlan_max in vlan_ranges: + res = { + 'id': uuidutils.generate_uuid(), + 'name': '', + 'default': True, + 'shared': True, + 'network_type': p_const.TYPE_VLAN, + 'physical_network': physical_network, + 'minimum': vlan_min, + 'maximum': vlan_max} + with db_api.CONTEXT_WRITER.using(ctx): + new_default_range_obj = ( + range_obj.NetworkSegmentRange(ctx, **res)) + new_default_range_obj.create() + + @db_api.retry_db_errors + def _delete_expired_default_network_segment_ranges(self): + ctx = context.get_admin_context() + with db_api.CONTEXT_WRITER.using(ctx): + filters = { + 'default': True, + 'network_type': p_const.TYPE_VLAN, + } + old_default_range_objs = range_obj.NetworkSegmentRange.get_objects( + ctx, **filters) + for obj in old_default_range_objs: + obj.delete() + def _parse_network_vlan_ranges(self): try: self.network_vlan_ranges = plugin_utils.parse_network_vlan_ranges( @@ -73,8 +113,9 @@ class VlanTypeDriver(helpers.SegmentTypeDriver): allocations[alloc.physical_network].append(alloc) # process vlan ranges for each configured physical network + ranges = self.get_network_segment_ranges() for (physical_network, - vlan_ranges) in self.network_vlan_ranges.items(): + vlan_ranges) in ranges.items(): # determine current configured allocatable vlans for # this physical network vlan_ids = set() @@ -132,21 +173,71 @@ class VlanTypeDriver(helpers.SegmentTypeDriver): alloc.physical_network}) alloc.delete() + @db_api.retry_db_errors + def _get_network_segment_ranges_from_db(self): + ranges = {} + ctx = context.get_admin_context() + with db_api.CONTEXT_READER.using(ctx): + range_objs = (range_obj.NetworkSegmentRange.get_objects( + ctx, network_type=self.get_type())) + for obj in range_objs: + physical_network = obj['physical_network'] + if physical_network not in ranges: + ranges[physical_network] = [] + ranges[physical_network].append((obj['minimum'], + obj['maximum'])) + return ranges + def get_type(self): return p_const.TYPE_VLAN def initialize(self): - self._sync_vlan_allocations() + if not range_plugin.is_network_segment_range_enabled(): + # service plugins are initialized/loaded after the ML2 driver + # initialization. Thus, we base on the information whether + # ``network_segment_range`` service plugin is enabled/defined in + # ``neutron.conf`` to decide whether to skip the first time sync + # allocation during driver initialization, instead of using the + # directory.get_plugin() method - the normal way used elsewhere to + # check if a plugin is loaded. + self._sync_vlan_allocations() LOG.info("VlanTypeDriver initialization complete") + def initialize_network_segment_range_support(self): + self._delete_expired_default_network_segment_ranges() + self._populate_new_default_network_segment_ranges() + # Override self.network_vlan_ranges with the network segment range + # information from DB and then do a sync_allocations since the + # segment range service plugin has not yet been loaded at this + # initialization time. + self.network_vlan_ranges = self._get_network_segment_ranges_from_db() + self._sync_vlan_allocations() + + def update_network_segment_range_allocations(self): + self._sync_vlan_allocations() + + def get_network_segment_ranges(self): + """Get the driver network segment ranges. + + Queries all VLAN network segment ranges from DB if the + ``NETWORK_SEGMENT_RANGE`` service plugin is enabled. Otherwise, + they will be loaded from the host config file - `ml2_conf.ini`. + """ + ranges = self.network_vlan_ranges + if directory.get_plugin(plugin_constants.NETWORK_SEGMENT_RANGE): + ranges = self._get_network_segment_ranges_from_db() + + return ranges + def is_partial_segment(self, segment): return segment.get(api.SEGMENTATION_ID) is None def validate_provider_segment(self, segment): physical_network = segment.get(api.PHYSICAL_NETWORK) segmentation_id = segment.get(api.SEGMENTATION_ID) + ranges = self.get_network_segment_ranges() if physical_network: - if physical_network not in self.network_vlan_ranges: + if physical_network not in ranges: msg = (_("physical_network '%s' unknown " "for VLAN provider network") % physical_network) raise exc.InvalidInput(error_message=msg) @@ -158,7 +249,7 @@ class VlanTypeDriver(helpers.SegmentTypeDriver): 'max': p_const.MAX_VLAN_TAG}) raise exc.InvalidInput(error_message=msg) else: - if not self.network_vlan_ranges.get(physical_network): + if not ranges.get(physical_network): msg = (_("Physical network %s requires segmentation_id " "to be specified when creating a provider " "network") % physical_network) @@ -175,7 +266,9 @@ class VlanTypeDriver(helpers.SegmentTypeDriver): msg = _("%s prohibited for VLAN provider network") % key raise exc.InvalidInput(error_message=msg) - def reserve_provider_segment(self, context, segment): + def reserve_provider_segment(self, context, segment, filters=None): + filters = filters or {} + project_id = filters.get('project_id') filters = {} physical_network = segment.get(api.PHYSICAL_NETWORK) if physical_network is not None: @@ -185,6 +278,9 @@ class VlanTypeDriver(helpers.SegmentTypeDriver): filters['vlan_id'] = vlan_id if self.is_partial_segment(segment): + if (directory.get_plugin( + plugin_constants.NETWORK_SEGMENT_RANGE)and project_id): + filters['project_id'] = project_id alloc = self.allocate_partially_specified_segment( context, **filters) if not alloc: @@ -200,10 +296,13 @@ class VlanTypeDriver(helpers.SegmentTypeDriver): api.SEGMENTATION_ID: alloc.vlan_id, api.MTU: self.get_mtu(alloc.physical_network)} - def allocate_tenant_segment(self, context): - for physnet in self.network_vlan_ranges: + def allocate_tenant_segment(self, context, filters=None): + filters = filters or {} + ranges = self.get_network_segment_ranges() + for physnet in ranges: + filters['physical_network'] = physnet alloc = self.allocate_partially_specified_segment( - context, physical_network=physnet) + context, **filters) if alloc: break else: @@ -217,7 +316,8 @@ class VlanTypeDriver(helpers.SegmentTypeDriver): physical_network = segment[api.PHYSICAL_NETWORK] vlan_id = segment[api.SEGMENTATION_ID] - ranges = self.network_vlan_ranges.get(physical_network, []) + vlan_ranges = self.get_network_segment_ranges() + ranges = vlan_ranges.get(physical_network, []) inside = any(lo <= vlan_id <= hi for lo, hi in ranges) count = False diff --git a/neutron/plugins/ml2/drivers/type_vxlan.py b/neutron/plugins/ml2/drivers/type_vxlan.py index 587b95ce461..4b77f35d256 100644 --- a/neutron/plugins/ml2/drivers/type_vxlan.py +++ b/neutron/plugins/ml2/drivers/type_vxlan.py @@ -19,6 +19,7 @@ from oslo_config import cfg from oslo_log import log from neutron.conf.plugins.ml2.drivers import driver_type +from neutron.db.models.plugins.ml2 import vxlanallocation as vxlan_alloc_model from neutron.objects.plugins.ml2 import vxlanallocation as vxlan_obj from neutron.plugins.ml2.drivers import type_tunnel @@ -32,6 +33,8 @@ class VxlanTypeDriver(type_tunnel.EndpointTunnelTypeDriver): def __init__(self): super(VxlanTypeDriver, self).__init__( vxlan_obj.VxlanAllocation, vxlan_obj.VxlanEndpoint) + self.model_segmentation_id = ( + vxlan_alloc_model.VxlanAllocation.vxlan_vni) def get_type(self): return p_const.TYPE_VXLAN diff --git a/neutron/plugins/ml2/managers.py b/neutron/plugins/ml2/managers.py index 5ca415d735c..af4e48c211a 100644 --- a/neutron/plugins/ml2/managers.py +++ b/neutron/plugins/ml2/managers.py @@ -187,6 +187,13 @@ class TypeManager(stevedore.named.NamedExtensionManager): LOG.info("Initializing driver for type '%s'", network_type) driver.obj.initialize() + def initialize_network_segment_range_support(self): + for network_type, driver in self.drivers.items(): + if network_type in constants.NETWORK_SEGMENT_RANGE_TYPES: + LOG.info("Initializing driver network segment range support " + "for type '%s'", network_type) + driver.obj.initialize_network_segment_range_support() + def _add_network_segment(self, context, network_id, segment, segment_index=0): segments_db.add_network_segment( @@ -195,20 +202,23 @@ class TypeManager(stevedore.named.NamedExtensionManager): def create_network_segments(self, context, network, tenant_id): """Call type drivers to create network segments.""" segments = self._process_provider_create(network) + filters = {'project_id': tenant_id} with db_api.CONTEXT_WRITER.using(context): network_id = network['id'] if segments: for segment_index, segment in enumerate(segments): segment = self.reserve_provider_segment( - context, segment) + context, segment, filters=filters) self._add_network_segment(context, network_id, segment, segment_index) elif (cfg.CONF.ml2.external_network_type and self._get_attribute(network, extnet_apidef.EXTERNAL)): - segment = self._allocate_ext_net_segment(context) + segment = self._allocate_ext_net_segment( + context, filters=filters) self._add_network_segment(context, network_id, segment) else: - segment = self._allocate_tenant_net_segment(context) + segment = self._allocate_tenant_net_segment( + context, filters=filters) self._add_network_segment(context, network_id, segment) def reserve_network_segment(self, context, segment_data): @@ -249,33 +259,33 @@ class TypeManager(stevedore.named.NamedExtensionManager): msg = _("network_type value '%s' not supported") % network_type raise exc.InvalidInput(error_message=msg) - def reserve_provider_segment(self, context, segment): + def reserve_provider_segment(self, context, segment, filters=None): network_type = segment.get(api.NETWORK_TYPE) driver = self.drivers.get(network_type) if isinstance(driver.obj, api.TypeDriver): return driver.obj.reserve_provider_segment(context.session, - segment) + segment, filters) else: return driver.obj.reserve_provider_segment(context, - segment) + segment, filters) - def _allocate_segment(self, context, network_type): + def _allocate_segment(self, context, network_type, filters=None): driver = self.drivers.get(network_type) if isinstance(driver.obj, api.TypeDriver): - return driver.obj.allocate_tenant_segment(context.session) + return driver.obj.allocate_tenant_segment(context.session, filters) else: - return driver.obj.allocate_tenant_segment(context) + return driver.obj.allocate_tenant_segment(context, filters) - def _allocate_tenant_net_segment(self, context): + def _allocate_tenant_net_segment(self, context, filters=None): for network_type in self.tenant_network_types: - segment = self._allocate_segment(context, network_type) + segment = self._allocate_segment(context, network_type, filters) if segment: return segment raise exc.NoNetworkAvailable() - def _allocate_ext_net_segment(self, context): + def _allocate_ext_net_segment(self, context, filters=None): network_type = cfg.CONF.ml2.external_network_type - segment = self._allocate_segment(context, network_type) + segment = self._allocate_segment(context, network_type, filters) if segment: return segment raise exc.NoNetworkAvailable() @@ -336,6 +346,13 @@ class TypeManager(stevedore.named.NamedExtensionManager): else: LOG.debug("No segment found with id %(segment_id)s", segment_id) + def update_network_segment_range_allocations(self, network_type): + driver = self.drivers.get(network_type) + driver.obj.update_network_segment_range_allocations() + + def network_type_supported(self, network_type): + return bool(network_type in self.drivers) + class MechanismManager(stevedore.named.NamedExtensionManager): """Manage networking mechanisms using drivers.""" diff --git a/neutron/services/network_segment_range/plugin.py b/neutron/services/network_segment_range/plugin.py index 5f1df3b8a17..72185f288a9 100644 --- a/neutron/services/network_segment_range/plugin.py +++ b/neutron/services/network_segment_range/plugin.py @@ -19,6 +19,7 @@ from neutron_lib import exceptions as lib_exc from neutron_lib.exceptions import network_segment_range as range_exc from neutron_lib.plugins import directory from neutron_lib.plugins import utils as plugin_utils +from oslo_config import cfg from oslo_log import helpers as log_helpers from oslo_log import log import six @@ -32,6 +33,13 @@ from neutron.objects import network_segment_range as obj_network_segment_range LOG = log.getLogger(__name__) +def is_network_segment_range_enabled(): + network_segment_range_class = ('neutron.services.network_segment_range.' + 'plugin.NetworkSegmentRangePlugin') + return any(p in cfg.CONF.service_plugins + for p in ['network_segment_range', network_segment_range_class]) + + class NetworkSegmentRangePlugin(ext_range.NetworkSegmentRangePluginBase): """Implements Neutron Network Segment Range Service plugin.""" diff --git a/neutron/tests/unit/plugins/ml2/drivers/base_type_tunnel.py b/neutron/tests/unit/plugins/ml2/drivers/base_type_tunnel.py index d5d65afb9be..cdf0971855c 100644 --- a/neutron/tests/unit/plugins/ml2/drivers/base_type_tunnel.py +++ b/neutron/tests/unit/plugins/ml2/drivers/base_type_tunnel.py @@ -23,6 +23,7 @@ from six import moves import testtools from testtools import matchers +from neutron.objects import network_segment_range as obj_network_segment_range from neutron.plugins.ml2.drivers import type_tunnel TUNNEL_IP_ONE = "10.10.10.10" @@ -32,8 +33,11 @@ HOST_ONE = 'fake_host_one' HOST_TWO = 'fake_host_two' TUN_MIN = 100 TUN_MAX = 109 +RAW_TUNNEL_RANGES = [str(TUN_MIN) + ':' + str(TUN_MAX)] TUNNEL_RANGES = [(TUN_MIN, TUN_MAX)] UPDATED_TUNNEL_RANGES = [(TUN_MIN + 5, TUN_MAX + 5)] +SERVICE_PLUGIN_KLASS = ('neutron.services.network_segment_range.plugin.' + 'NetworkSegmentRangePlugin') class TunnelTypeTestMixin(object): @@ -460,3 +464,42 @@ class TunnelTypeMTUTestMixin(object): def test_get_mtu_ipv6(self): self._test_get_mtu(6) + + +class TunnelTypeNetworkSegmentRangeTestMixin(object): + + DRIVER_CLASS = None + + def setUp(self): + super(TunnelTypeNetworkSegmentRangeTestMixin, self).setUp() + cfg.CONF.set_override('service_plugins', [SERVICE_PLUGIN_KLASS]) + self.context = context.Context() + self.driver = self.DRIVER_CLASS() + + def test__populate_new_default_network_segment_ranges(self): + # _populate_new_default_network_segment_ranges will be called when + # the type driver initializes with `network_segment_range` loaded as + # one of the `service_plugins` + self.driver._initialize(RAW_TUNNEL_RANGES) + self.driver.initialize_network_segment_range_support() + self.driver.sync_allocations() + ret = obj_network_segment_range.NetworkSegmentRange.get_objects( + self.context) + self.assertEqual(1, len(ret)) + network_segment_range = ret[0] + self.assertTrue(network_segment_range.default) + self.assertTrue(network_segment_range.shared) + self.assertIsNone(network_segment_range.project_id) + self.assertEqual(self.driver.get_type(), + network_segment_range.network_type) + self.assertIsNone(network_segment_range.physical_network) + self.assertEqual(TUN_MIN, network_segment_range.minimum) + self.assertEqual(TUN_MAX, network_segment_range.maximum) + + def test__delete_expired_default_network_segment_ranges(self): + self.driver.tunnel_ranges = TUNNEL_RANGES + self.driver.sync_allocations() + self.driver._delete_expired_default_network_segment_ranges() + ret = obj_network_segment_range.NetworkSegmentRange.get_objects( + self.context) + self.assertEqual(0, len(ret)) diff --git a/neutron/tests/unit/plugins/ml2/drivers/test_helpers.py b/neutron/tests/unit/plugins/ml2/drivers/test_helpers.py index 8b9cddd1d7b..739371cfd3c 100644 --- a/neutron/tests/unit/plugins/ml2/drivers/test_helpers.py +++ b/neutron/tests/unit/plugins/ml2/drivers/test_helpers.py @@ -15,6 +15,8 @@ import mock from neutron_lib import context +from neutron_lib.plugins import utils as plugin_utils +from oslo_config import cfg from oslo_db import exception as exc from sqlalchemy.orm import query @@ -29,6 +31,10 @@ VLAN_OUTSIDE = 100 NETWORK_VLAN_RANGES = { TENANT_NET: [(VLAN_MIN, VLAN_MAX)], } +NETWORK_VLAN_RANGES_CFG_ENTRIES = [TENANT_NET, "%s:%s:%s" % + (TENANT_NET, VLAN_MIN, VLAN_MAX)] +SERVICE_PLUGIN_KLASS = ('neutron.services.network_segment_range.plugin.' + 'NetworkSegmentRangePlugin') class HelpersTest(testlib_api.SqlTestCase): @@ -131,3 +137,19 @@ class HelpersTest(testlib_api.SqlTestCase): observed = self.driver.allocate_partially_specified_segment( self.context, **expected) self.check_raw_segment(expected, observed) + + +class HelpersTestWithNetworkSegmentRange(HelpersTest): + + def setUp(self): + super(HelpersTestWithNetworkSegmentRange, self).setUp() + cfg.CONF.set_override('network_vlan_ranges', + NETWORK_VLAN_RANGES_CFG_ENTRIES, + group='ml2_type_vlan') + cfg.CONF.set_override('service_plugins', [SERVICE_PLUGIN_KLASS]) + self.network_vlan_ranges = plugin_utils.parse_network_vlan_ranges( + NETWORK_VLAN_RANGES_CFG_ENTRIES) + self.context = context.get_admin_context() + self.driver = type_vlan.VlanTypeDriver() + self.driver.initialize_network_segment_range_support() + self.driver._sync_vlan_allocations() diff --git a/neutron/tests/unit/plugins/ml2/drivers/test_type_geneve.py b/neutron/tests/unit/plugins/ml2/drivers/test_type_geneve.py index 61180a017d3..872dd55730e 100644 --- a/neutron/tests/unit/plugins/ml2/drivers/test_type_geneve.py +++ b/neutron/tests/unit/plugins/ml2/drivers/test_type_geneve.py @@ -28,6 +28,7 @@ HOST_TWO = 'fake_host_two2' class GeneveTypeTest(base_type_tunnel.TunnelTypeTestMixin, + base_type_tunnel.TunnelTypeNetworkSegmentRangeTestMixin, testlib_api.SqlTestCase): DRIVER_CLASS = type_geneve.GeneveTypeDriver TYPE = p_const.TYPE_GENEVE diff --git a/neutron/tests/unit/plugins/ml2/drivers/test_type_gre.py b/neutron/tests/unit/plugins/ml2/drivers/test_type_gre.py index 2670e2cebba..61bc3de9dbd 100644 --- a/neutron/tests/unit/plugins/ml2/drivers/test_type_gre.py +++ b/neutron/tests/unit/plugins/ml2/drivers/test_type_gre.py @@ -28,6 +28,7 @@ HOST_TWO = 'fake_host_two' class GreTypeTest(base_type_tunnel.TunnelTypeTestMixin, + base_type_tunnel.TunnelTypeNetworkSegmentRangeTestMixin, testlib_api.SqlTestCase): DRIVER_MODULE = type_gre DRIVER_CLASS = type_gre.GreTypeDriver diff --git a/neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py b/neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py index f8af6331ded..657edea9d85 100644 --- a/neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py +++ b/neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py @@ -23,6 +23,7 @@ from neutron_lib.plugins import utils as plugin_utils from oslo_config import cfg from testtools import matchers +from neutron.objects import network_segment_range as obj_network_segment_range from neutron.objects.plugins.ml2 import vlanallocation as vlan_alloc_obj from neutron.plugins.ml2.drivers import type_vlan from neutron.tests.unit import testlib_api @@ -41,6 +42,8 @@ EMPTY_VLAN_RANGES = { PROVIDER_NET: [] } CORE_PLUGIN = 'ml2' +SERVICE_PLUGIN_KLASS = ('neutron.services.network_segment_range.plugin.' + 'NetworkSegmentRangePlugin') class VlanTypeTest(testlib_api.SqlTestCase): @@ -319,3 +322,41 @@ class VlanTypeAllocationTest(testlib_api.SqlTestCase): driver.allocate_tenant_segment(ctx)) # then nothing self.assertFalse(driver.allocate_tenant_segment(ctx)) + + +class VlanTypeTestWithNetworkSegmentRange(testlib_api.SqlTestCase): + + def setUp(self): + super(VlanTypeTestWithNetworkSegmentRange, self).setUp() + cfg.CONF.set_override('network_vlan_ranges', + NETWORK_VLAN_RANGES, + group='ml2_type_vlan') + cfg.CONF.set_override('service_plugins', [SERVICE_PLUGIN_KLASS]) + self.network_vlan_ranges = plugin_utils.parse_network_vlan_ranges( + NETWORK_VLAN_RANGES) + self.driver = type_vlan.VlanTypeDriver() + self.driver._sync_vlan_allocations() + self.context = context.Context() + self.setup_coreplugin(CORE_PLUGIN) + + def test__populate_new_default_network_segment_ranges(self): + # _populate_new_default_network_segment_ranges will be called when + # the type driver initializes with `network_segment_range` loaded as + # one of the `service_plugins` + ret = obj_network_segment_range.NetworkSegmentRange.get_objects( + self.context) + self.assertEqual(1, len(ret)) + network_segment_range = ret[0] + self.assertTrue(network_segment_range.default) + self.assertTrue(network_segment_range.shared) + self.assertIsNone(network_segment_range.project_id) + self.assertEqual(p_const.TYPE_VLAN, network_segment_range.network_type) + self.assertEqual(TENANT_NET, network_segment_range.physical_network) + self.assertEqual(VLAN_MIN, network_segment_range.minimum) + self.assertEqual(VLAN_MAX, network_segment_range.maximum) + + def test__delete_expired_default_network_segment_ranges(self): + self.driver._delete_expired_default_network_segment_ranges() + ret = obj_network_segment_range.NetworkSegmentRange.get_objects( + self.context) + self.assertEqual(0, len(ret)) diff --git a/neutron/tests/unit/plugins/ml2/drivers/test_type_vxlan.py b/neutron/tests/unit/plugins/ml2/drivers/test_type_vxlan.py index c81906149ca..ed637829f78 100644 --- a/neutron/tests/unit/plugins/ml2/drivers/test_type_vxlan.py +++ b/neutron/tests/unit/plugins/ml2/drivers/test_type_vxlan.py @@ -26,6 +26,7 @@ VXLAN_UDP_PORT_TWO = 8888 class VxlanTypeTest(base_type_tunnel.TunnelTypeTestMixin, + base_type_tunnel.TunnelTypeNetworkSegmentRangeTestMixin, testlib_api.SqlTestCase): DRIVER_MODULE = type_vxlan DRIVER_CLASS = type_vxlan.VxlanTypeDriver