From be1a0daab0d60411f4cc5a0ce92030cc07bfcbdf Mon Sep 17 00:00:00 2001 From: Rodolfo Alonso Hernandez Date: Tue, 23 Mar 2021 18:57:31 +0000 Subject: [PATCH] Randomize segmentation ID assignation If plugin "network_segment_range" is not enabled and a new segment is required, if no segmentation ID is provided in the request, the segmentation ID assigned is randomly retrieved from the non allocated segmentation IDs. The goal is to improve the concurrent network (and segment) creation. If several segments are created in parallel, this random query will return a different segmentation ID to each one, avoiding the database retry request. Closes-Bug: #1920923 Conflicts: neutron/common/utils.py neutron/plugins/ml2/drivers/helpers.py neutron/tests/functional/objects/plugins/ml2/test_base.py neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py Change-Id: Id3f71611a00e69c4f22340ca4d05d95e4373cf69 (cherry picked from commit 6eaa6d83d7c7f07fd4bf04879c91582de504eff4) (cherry picked from commit ab56a5cd652f57890723d4df5ba6ed22845070fa) --- neutron/common/utils.py | 17 ++++ neutron/objects/plugins/ml2/base.py | 10 ++- neutron/plugins/ml2/drivers/helpers.py | 12 ++- .../functional/objects/plugins/__init__.py | 0 .../objects/plugins/ml2/__init__.py | 0 .../objects/plugins/ml2/test_base.py | 90 +++++++++++++++++++ .../plugins/ml2/test_geneveallocation.py | 26 ++++++ .../objects/plugins/ml2/test_greallocation.py | 26 ++++++ .../plugins/ml2/test_vlanallocation.py | 26 ++++++ .../plugins/ml2/test_vxlanallocation.py | 26 ++++++ .../unit/objects/plugins/ml2/test_base.py | 16 ++-- 11 files changed, 235 insertions(+), 14 deletions(-) create mode 100644 neutron/tests/functional/objects/plugins/__init__.py create mode 100644 neutron/tests/functional/objects/plugins/ml2/__init__.py create mode 100644 neutron/tests/functional/objects/plugins/ml2/test_base.py create mode 100644 neutron/tests/functional/objects/plugins/ml2/test_geneveallocation.py create mode 100644 neutron/tests/functional/objects/plugins/ml2/test_greallocation.py create mode 100644 neutron/tests/functional/objects/plugins/ml2/test_vlanallocation.py create mode 100644 neutron/tests/functional/objects/plugins/ml2/test_vxlanallocation.py diff --git a/neutron/common/utils.py b/neutron/common/utils.py index 4bed9b0e14d..d947b203bac 100644 --- a/neutron/common/utils.py +++ b/neutron/common/utils.py @@ -48,6 +48,10 @@ from oslo_utils import timeutils from oslo_utils import uuidutils from osprofiler import profiler import pkg_resources +from sqlalchemy.dialects.mysql import dialect as mysql_dialect +from sqlalchemy.dialects.postgresql import dialect as postgresql_dialect +from sqlalchemy.dialects.sqlite import dialect as sqlite_dialect +from sqlalchemy.sql.expression import func as sql_func import neutron from neutron._i18n import _ @@ -1016,3 +1020,16 @@ def skip_exceptions(exceptions): ctx.reraise = False return wrapper return decorator + + +def get_sql_random_method(sql_dialect_name): + """Return the SQL random method supported depending on the dialect.""" + # NOTE(ralonsoh): this method is a good candidate to be implemented in + # oslo.db. + # https://www.postgresql.org/docs/8.2/functions-math.html + # https://www.sqlite.org/c3ref/randomness.html + if sql_dialect_name in (postgresql_dialect.name, sqlite_dialect.name): + return sql_func.random + # https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html + elif sql_dialect_name == mysql_dialect.name: + return sql_func.rand diff --git a/neutron/objects/plugins/ml2/base.py b/neutron/objects/plugins/ml2/base.py index e173b909445..f7540b39af3 100644 --- a/neutron/objects/plugins/ml2/base.py +++ b/neutron/objects/plugins/ml2/base.py @@ -17,7 +17,7 @@ import abc import netaddr import six -from neutron.common import _constants as common_constants +from neutron.common import utils as n_utils from neutron.objects import base @@ -44,14 +44,18 @@ class EndpointBase(base.NeutronDbObject): class SegmentAllocation(object): @classmethod - def get_unallocated_segments(cls, context, **filters): + def get_random_unallocated_segment(cls, context, **filters): with cls.db_context_reader(context): columns = set(dict(cls.db_model.__table__.columns)) model_filters = dict((k, filters[k]) for k in columns & set(filters.keys())) query = context.session.query(cls.db_model).filter_by( allocated=False, **model_filters) - return query.limit(common_constants.IDPOOL_SELECT_SIZE).all() + rand_func = n_utils.get_sql_random_method( + context.session.bind.dialect.name) + if rand_func: + query = query.order_by(rand_func()) + return query.first() @classmethod def allocate(cls, context, **segment): diff --git a/neutron/plugins/ml2/drivers/helpers.py b/neutron/plugins/ml2/drivers/helpers.py index 9e3d97c861a..9db8f77fb9c 100644 --- a/neutron/plugins/ml2/drivers/helpers.py +++ b/neutron/plugins/ml2/drivers/helpers.py @@ -137,17 +137,23 @@ class SegmentTypeDriver(BaseTypeDriver): self.model_segmentation_id, **filters)] else: calls = [functools.partial( - self.segmentation_obj.get_unallocated_segments, + self.segmentation_obj.get_random_unallocated_segment, context, **filters)] + try_to_allocate = False for call in calls: allocations = call() + if not isinstance(allocations, list): + allocations = [allocations] if allocations else [] for alloc in allocations: segment = dict((k, alloc[k]) for k in self.primary_keys) + try_to_allocate = True if self.segmentation_obj.allocate(context, **segment): LOG.debug('%(type)s segment allocate from pool success ' 'with %(segment)s ', {'type': network_type, 'segment': segment}) return alloc - raise db_exc.RetryRequest( - exceptions.NoNetworkFoundInMaximumAllowedAttempts()) + + if try_to_allocate: + raise db_exc.RetryRequest( + exceptions.NoNetworkFoundInMaximumAllowedAttempts()) diff --git a/neutron/tests/functional/objects/plugins/__init__.py b/neutron/tests/functional/objects/plugins/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/neutron/tests/functional/objects/plugins/ml2/__init__.py b/neutron/tests/functional/objects/plugins/ml2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/neutron/tests/functional/objects/plugins/ml2/test_base.py b/neutron/tests/functional/objects/plugins/ml2/test_base.py new file mode 100644 index 00000000000..8ef0788d99a --- /dev/null +++ b/neutron/tests/functional/objects/plugins/ml2/test_base.py @@ -0,0 +1,90 @@ +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools + +from neutron_lib import context + +from neutron.tests.unit import testlib_api + + +class _SegmentAllocation(testlib_api.SqlTestCase): + + PHYSNETS = ('phys1', 'phys2') + NUM_SEGIDS = 10 + segment_allocation_class = None + + def setUp(self): + if not self.segment_allocation_class: + self.skipTest('No allocation class defined') + super(_SegmentAllocation, self).setUp() + self.context = context.Context(user_id='usier_id', + tenant_id='tenant_id') + self.segid_field = ( + self.segment_allocation_class.get_segmentation_id().name) + self.is_vlan = ('physical_network' in + self.segment_allocation_class.db_model.primary_keys()) + pk_columns = self.segment_allocation_class.db_model.__table__.\ + primary_key.columns + self.primary_keys = {col.name for col in pk_columns} + self.segments = None + + def _create_segments(self, num_segids, physnets, allocated=False): + + if self.is_vlan: + self.segments = list(itertools.product(physnets, + range(1, num_segids + 1))) + kwargs_list = [ + {'physical_network': physnet, + self.segid_field: segid, + 'allocated': allocated} for physnet, segid in self.segments] + else: + self.segments = list(range(1, num_segids + 1)) + kwargs_list = [{self.segid_field: segid, + 'allocated': allocated} for segid in self.segments] + + for kwargs in kwargs_list: + self.segment_allocation_class(self.context, **kwargs).create() + + self.assertTrue( + len(kwargs_list), + len(self.segment_allocation_class.get_objects(self.context))) + + def test_get_random_unallocated_segment_and_allocate(self): + m_get = self.segment_allocation_class.get_random_unallocated_segment + m_alloc = self.segment_allocation_class.allocate + self._create_segments(self.NUM_SEGIDS, self.PHYSNETS) + for _ in range(len(self.segments)): + unalloc = m_get(self.context) + segment = dict((k, unalloc[k]) for k in self.primary_keys) + m_alloc(self.context, **segment) + if self.is_vlan: + self.segments.remove((unalloc['physical_network'], + unalloc.segmentation_id)) + else: + self.segments.remove(unalloc.segmentation_id) + + self.assertEqual(0, len(self.segments)) + self.assertIsNone(m_get(self.context)) + + +class _SegmentAllocationMySQL(_SegmentAllocation, + testlib_api.MySQLTestCaseMixin): + pass + + +class _SegmentAllocationPostgreSQL(_SegmentAllocation, + testlib_api.PostgreSQLTestCaseMixin): + pass diff --git a/neutron/tests/functional/objects/plugins/ml2/test_geneveallocation.py b/neutron/tests/functional/objects/plugins/ml2/test_geneveallocation.py new file mode 100644 index 00000000000..19a82b162d8 --- /dev/null +++ b/neutron/tests/functional/objects/plugins/ml2/test_geneveallocation.py @@ -0,0 +1,26 @@ +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron.objects.plugins.ml2 import geneveallocation +from neutron.tests.functional.objects.plugins.ml2 import test_base + + +class TestGeneveSegmentAllocationMySQL(test_base._SegmentAllocationMySQL): + segment_allocation_class = geneveallocation.GeneveAllocation + + +class TestGeneveSegmentAllocationPostgreSQL( + test_base._SegmentAllocationPostgreSQL): + segment_allocation_class = geneveallocation.GeneveAllocation diff --git a/neutron/tests/functional/objects/plugins/ml2/test_greallocation.py b/neutron/tests/functional/objects/plugins/ml2/test_greallocation.py new file mode 100644 index 00000000000..deca6667e71 --- /dev/null +++ b/neutron/tests/functional/objects/plugins/ml2/test_greallocation.py @@ -0,0 +1,26 @@ +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron.objects.plugins.ml2 import greallocation +from neutron.tests.functional.objects.plugins.ml2 import test_base + + +class TestGreSegmentAllocationMySQL(test_base._SegmentAllocationMySQL): + segment_allocation_class = greallocation.GreAllocation + + +class TestGreSegmentAllocationPostgreSQL( + test_base._SegmentAllocationPostgreSQL): + segment_allocation_class = greallocation.GreAllocation diff --git a/neutron/tests/functional/objects/plugins/ml2/test_vlanallocation.py b/neutron/tests/functional/objects/plugins/ml2/test_vlanallocation.py new file mode 100644 index 00000000000..f42c6384dfb --- /dev/null +++ b/neutron/tests/functional/objects/plugins/ml2/test_vlanallocation.py @@ -0,0 +1,26 @@ +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron.objects.plugins.ml2 import vlanallocation +from neutron.tests.functional.objects.plugins.ml2 import test_base + + +class TestVlanSegmentAllocationMySQL(test_base._SegmentAllocationMySQL): + segment_allocation_class = vlanallocation.VlanAllocation + + +class TestVlanSegmentAllocationPostgreSQL( + test_base._SegmentAllocationPostgreSQL): + segment_allocation_class = vlanallocation.VlanAllocation diff --git a/neutron/tests/functional/objects/plugins/ml2/test_vxlanallocation.py b/neutron/tests/functional/objects/plugins/ml2/test_vxlanallocation.py new file mode 100644 index 00000000000..ff6109e6a34 --- /dev/null +++ b/neutron/tests/functional/objects/plugins/ml2/test_vxlanallocation.py @@ -0,0 +1,26 @@ +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron.objects.plugins.ml2 import vxlanallocation +from neutron.tests.functional.objects.plugins.ml2 import test_base + + +class TestVxlanSegmentAllocationMySQL(test_base._SegmentAllocationMySQL): + segment_allocation_class = vxlanallocation.VxlanAllocation + + +class TestVxlanSegmentAllocationPostgreSQL( + test_base._SegmentAllocationPostgreSQL): + segment_allocation_class = vxlanallocation.VxlanAllocation diff --git a/neutron/tests/unit/objects/plugins/ml2/test_base.py b/neutron/tests/unit/objects/plugins/ml2/test_base.py index d7993d62d58..3ef9ae0edec 100644 --- a/neutron/tests/unit/objects/plugins/ml2/test_base.py +++ b/neutron/tests/unit/objects/plugins/ml2/test_base.py @@ -16,19 +16,19 @@ class SegmentAllocationDbObjTestCase(object): - def test_get_unallocated_segments(self): - self.assertEqual( - [], self._test_class.get_unallocated_segments(self.context)) + def test_get_random_unallocated_segment(self): + self.assertIsNone( + self._test_class.get_random_unallocated_segment(self.context)) obj = self.objs[0] obj.allocated = True obj.create() - self.assertEqual( - [], self._test_class.get_unallocated_segments(self.context)) + self.assertIsNone( + self._test_class.get_random_unallocated_segment(self.context)) obj = self.objs[1] obj.allocated = False obj.create() - allocations = self._test_class.get_unallocated_segments(self.context) - self.assertEqual(1, len(allocations)) - self.assertEqual(obj.segmentation_id, allocations[0].segmentation_id) + allocations = self._test_class.get_random_unallocated_segment( + self.context) + self.assertEqual(obj.segmentation_id, allocations.segmentation_id)