Randomize segmentation ID assignation

If plugin "network_segment_range" is not enabled and a new segment
is required, if no segmentation ID is provided in the request, the
segmentation ID assigned is randomly retrieved from the non
allocated segmentation IDs.

The goal is to improve the concurrent network (and segment) creation.
If several segments are created in parallel, this random query
will return a different segmentation ID to each one, avoiding the
database retry request.

Closes-Bug: #1920923

Conflicts:
    neutron/common/utils.py
    neutron/plugins/ml2/drivers/helpers.py
    neutron/tests/functional/objects/plugins/ml2/test_base.py
    neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py

Change-Id: Id3f71611a00e69c4f22340ca4d05d95e4373cf69
(cherry picked from commit 6eaa6d83d7)
(cherry picked from commit ab56a5cd65)
This commit is contained in:
Rodolfo Alonso Hernandez 2021-03-23 18:57:31 +00:00 committed by Slawek Kaplonski
parent 68b7aaf500
commit be1a0daab0
11 changed files with 235 additions and 14 deletions

View File

@ -48,6 +48,10 @@ from oslo_utils import timeutils
from oslo_utils import uuidutils from oslo_utils import uuidutils
from osprofiler import profiler from osprofiler import profiler
import pkg_resources import pkg_resources
from sqlalchemy.dialects.mysql import dialect as mysql_dialect
from sqlalchemy.dialects.postgresql import dialect as postgresql_dialect
from sqlalchemy.dialects.sqlite import dialect as sqlite_dialect
from sqlalchemy.sql.expression import func as sql_func
import neutron import neutron
from neutron._i18n import _ from neutron._i18n import _
@ -1016,3 +1020,16 @@ def skip_exceptions(exceptions):
ctx.reraise = False ctx.reraise = False
return wrapper return wrapper
return decorator return decorator
def get_sql_random_method(sql_dialect_name):
"""Return the SQL random method supported depending on the dialect."""
# NOTE(ralonsoh): this method is a good candidate to be implemented in
# oslo.db.
# https://www.postgresql.org/docs/8.2/functions-math.html
# https://www.sqlite.org/c3ref/randomness.html
if sql_dialect_name in (postgresql_dialect.name, sqlite_dialect.name):
return sql_func.random
# https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html
elif sql_dialect_name == mysql_dialect.name:
return sql_func.rand

View File

@ -17,7 +17,7 @@ import abc
import netaddr import netaddr
import six import six
from neutron.common import _constants as common_constants from neutron.common import utils as n_utils
from neutron.objects import base from neutron.objects import base
@ -44,14 +44,18 @@ class EndpointBase(base.NeutronDbObject):
class SegmentAllocation(object): class SegmentAllocation(object):
@classmethod @classmethod
def get_unallocated_segments(cls, context, **filters): def get_random_unallocated_segment(cls, context, **filters):
with cls.db_context_reader(context): with cls.db_context_reader(context):
columns = set(dict(cls.db_model.__table__.columns)) columns = set(dict(cls.db_model.__table__.columns))
model_filters = dict((k, filters[k]) model_filters = dict((k, filters[k])
for k in columns & set(filters.keys())) for k in columns & set(filters.keys()))
query = context.session.query(cls.db_model).filter_by( query = context.session.query(cls.db_model).filter_by(
allocated=False, **model_filters) allocated=False, **model_filters)
return query.limit(common_constants.IDPOOL_SELECT_SIZE).all() rand_func = n_utils.get_sql_random_method(
context.session.bind.dialect.name)
if rand_func:
query = query.order_by(rand_func())
return query.first()
@classmethod @classmethod
def allocate(cls, context, **segment): def allocate(cls, context, **segment):

View File

@ -137,17 +137,23 @@ class SegmentTypeDriver(BaseTypeDriver):
self.model_segmentation_id, **filters)] self.model_segmentation_id, **filters)]
else: else:
calls = [functools.partial( calls = [functools.partial(
self.segmentation_obj.get_unallocated_segments, self.segmentation_obj.get_random_unallocated_segment,
context, **filters)] context, **filters)]
try_to_allocate = False
for call in calls: for call in calls:
allocations = call() allocations = call()
if not isinstance(allocations, list):
allocations = [allocations] if allocations else []
for alloc in allocations: for alloc in allocations:
segment = dict((k, alloc[k]) for k in self.primary_keys) segment = dict((k, alloc[k]) for k in self.primary_keys)
try_to_allocate = True
if self.segmentation_obj.allocate(context, **segment): if self.segmentation_obj.allocate(context, **segment):
LOG.debug('%(type)s segment allocate from pool success ' LOG.debug('%(type)s segment allocate from pool success '
'with %(segment)s ', {'type': network_type, 'with %(segment)s ', {'type': network_type,
'segment': segment}) 'segment': segment})
return alloc return alloc
raise db_exc.RetryRequest(
exceptions.NoNetworkFoundInMaximumAllowedAttempts()) if try_to_allocate:
raise db_exc.RetryRequest(
exceptions.NoNetworkFoundInMaximumAllowedAttempts())

View File

@ -0,0 +1,90 @@
# Copyright 2021 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
from neutron_lib import context
from neutron.tests.unit import testlib_api
class _SegmentAllocation(testlib_api.SqlTestCase):
PHYSNETS = ('phys1', 'phys2')
NUM_SEGIDS = 10
segment_allocation_class = None
def setUp(self):
if not self.segment_allocation_class:
self.skipTest('No allocation class defined')
super(_SegmentAllocation, self).setUp()
self.context = context.Context(user_id='usier_id',
tenant_id='tenant_id')
self.segid_field = (
self.segment_allocation_class.get_segmentation_id().name)
self.is_vlan = ('physical_network' in
self.segment_allocation_class.db_model.primary_keys())
pk_columns = self.segment_allocation_class.db_model.__table__.\
primary_key.columns
self.primary_keys = {col.name for col in pk_columns}
self.segments = None
def _create_segments(self, num_segids, physnets, allocated=False):
if self.is_vlan:
self.segments = list(itertools.product(physnets,
range(1, num_segids + 1)))
kwargs_list = [
{'physical_network': physnet,
self.segid_field: segid,
'allocated': allocated} for physnet, segid in self.segments]
else:
self.segments = list(range(1, num_segids + 1))
kwargs_list = [{self.segid_field: segid,
'allocated': allocated} for segid in self.segments]
for kwargs in kwargs_list:
self.segment_allocation_class(self.context, **kwargs).create()
self.assertTrue(
len(kwargs_list),
len(self.segment_allocation_class.get_objects(self.context)))
def test_get_random_unallocated_segment_and_allocate(self):
m_get = self.segment_allocation_class.get_random_unallocated_segment
m_alloc = self.segment_allocation_class.allocate
self._create_segments(self.NUM_SEGIDS, self.PHYSNETS)
for _ in range(len(self.segments)):
unalloc = m_get(self.context)
segment = dict((k, unalloc[k]) for k in self.primary_keys)
m_alloc(self.context, **segment)
if self.is_vlan:
self.segments.remove((unalloc['physical_network'],
unalloc.segmentation_id))
else:
self.segments.remove(unalloc.segmentation_id)
self.assertEqual(0, len(self.segments))
self.assertIsNone(m_get(self.context))
class _SegmentAllocationMySQL(_SegmentAllocation,
testlib_api.MySQLTestCaseMixin):
pass
class _SegmentAllocationPostgreSQL(_SegmentAllocation,
testlib_api.PostgreSQLTestCaseMixin):
pass

View File

@ -0,0 +1,26 @@
# Copyright 2021 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from neutron.objects.plugins.ml2 import geneveallocation
from neutron.tests.functional.objects.plugins.ml2 import test_base
class TestGeneveSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
segment_allocation_class = geneveallocation.GeneveAllocation
class TestGeneveSegmentAllocationPostgreSQL(
test_base._SegmentAllocationPostgreSQL):
segment_allocation_class = geneveallocation.GeneveAllocation

View File

@ -0,0 +1,26 @@
# Copyright 2021 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from neutron.objects.plugins.ml2 import greallocation
from neutron.tests.functional.objects.plugins.ml2 import test_base
class TestGreSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
segment_allocation_class = greallocation.GreAllocation
class TestGreSegmentAllocationPostgreSQL(
test_base._SegmentAllocationPostgreSQL):
segment_allocation_class = greallocation.GreAllocation

View File

@ -0,0 +1,26 @@
# Copyright 2021 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from neutron.objects.plugins.ml2 import vlanallocation
from neutron.tests.functional.objects.plugins.ml2 import test_base
class TestVlanSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
segment_allocation_class = vlanallocation.VlanAllocation
class TestVlanSegmentAllocationPostgreSQL(
test_base._SegmentAllocationPostgreSQL):
segment_allocation_class = vlanallocation.VlanAllocation

View File

@ -0,0 +1,26 @@
# Copyright 2021 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from neutron.objects.plugins.ml2 import vxlanallocation
from neutron.tests.functional.objects.plugins.ml2 import test_base
class TestVxlanSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
segment_allocation_class = vxlanallocation.VxlanAllocation
class TestVxlanSegmentAllocationPostgreSQL(
test_base._SegmentAllocationPostgreSQL):
segment_allocation_class = vxlanallocation.VxlanAllocation

View File

@ -16,19 +16,19 @@
class SegmentAllocationDbObjTestCase(object): class SegmentAllocationDbObjTestCase(object):
def test_get_unallocated_segments(self): def test_get_random_unallocated_segment(self):
self.assertEqual( self.assertIsNone(
[], self._test_class.get_unallocated_segments(self.context)) self._test_class.get_random_unallocated_segment(self.context))
obj = self.objs[0] obj = self.objs[0]
obj.allocated = True obj.allocated = True
obj.create() obj.create()
self.assertEqual( self.assertIsNone(
[], self._test_class.get_unallocated_segments(self.context)) self._test_class.get_random_unallocated_segment(self.context))
obj = self.objs[1] obj = self.objs[1]
obj.allocated = False obj.allocated = False
obj.create() obj.create()
allocations = self._test_class.get_unallocated_segments(self.context) allocations = self._test_class.get_random_unallocated_segment(
self.assertEqual(1, len(allocations)) self.context)
self.assertEqual(obj.segmentation_id, allocations[0].segmentation_id) self.assertEqual(obj.segmentation_id, allocations.segmentation_id)