Randomize segmentation ID assignation

If plugin "network_segment_range" is not enabled and a new segment
is required, if no segmentation ID is provided in the request, the
segmentation ID assigned is randomly retrieved from the non
allocated segmentation IDs.

The goal is to improve the concurrent network (and segment) creation.
If several segments are created in parallel, this random query
will return a different segmentation ID to each one, avoiding the
database retry request.

Closes-Bug: #1920923

Change-Id: Id3f71611a00e69c4f22340ca4d05d95e4373cf69

neutron/common/utils.py

@@ -48,6 +48,10 @@ from oslo_utils import timeutils
 from oslo_utils import uuidutils
 from osprofiler import profiler
 import pkg_resources
+from sqlalchemy.dialects.mysql import dialect as mysql_dialect
+from sqlalchemy.dialects.postgresql import dialect as postgresql_dialect
+from sqlalchemy.dialects.sqlite import dialect as sqlite_dialect
+from sqlalchemy.sql.expression import func as sql_func
 
 import neutron
 from neutron._i18n import _
@@ -1047,3 +1051,16 @@ def get_elevated_context(context):
     if cfg.CONF.oslo_policy.enforce_new_defaults:
         admin_context.system_scope = 'all'
     return admin_context
+
+
+def get_sql_random_method(sql_dialect_name):
+    """Return the SQL random method supported depending on the dialect."""
+    # NOTE(ralonsoh): this method is a good candidate to be implemented in
+    # oslo.db.
+    # https://www.postgresql.org/docs/8.2/functions-math.html
+    # https://www.sqlite.org/c3ref/randomness.html
+    if sql_dialect_name in (postgresql_dialect.name, sqlite_dialect.name):
+        return sql_func.random
+    # https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html
+    elif sql_dialect_name == mysql_dialect.name:
+        return sql_func.rand

neutron/objects/plugins/ml2/base.py

@@ -16,7 +16,7 @@ import abc
 
 import netaddr
 
-from neutron.common import _constants as common_constants
+from neutron.common import utils as n_utils
 from neutron.objects import base
 
 
@@ -42,14 +42,18 @@ class EndpointBase(base.NeutronDbObject):
 class SegmentAllocation(object, metaclass=abc.ABCMeta):
 
     @classmethod
-    def get_unallocated_segments(cls, context, **filters):
+    def get_random_unallocated_segment(cls, context, **filters):
         with cls.db_context_reader(context):
             columns = set(dict(cls.db_model.__table__.columns))
             model_filters = dict((k, filters[k])
                                  for k in columns & set(filters.keys()))
             query = context.session.query(cls.db_model).filter_by(
                 allocated=False, **model_filters)
-            return query.limit(common_constants.IDPOOL_SELECT_SIZE).all()
+            rand_func = n_utils.get_sql_random_method(
+                context.session.bind.dialect.name)
+            if rand_func:
+                query = query.order_by(rand_func())
+            return query.first()
 
     @classmethod
     def allocate(cls, context, **segment):

neutron/plugins/ml2/drivers/helpers.py

@@ -138,20 +138,26 @@ class SegmentTypeDriver(BaseTypeDriver):
                     self.model_segmentation_id, **filters)]
         else:
             calls = [functools.partial(
-                self.segmentation_obj.get_unallocated_segments,
+                self.segmentation_obj.get_random_unallocated_segment,
                 context, **filters)]
 
+        try_to_allocate = False
         for call in calls:
             allocations = call()
+            if not isinstance(allocations, list):
+                allocations = [allocations] if allocations else []
             for alloc in allocations:
                 segment = dict((k, alloc[k]) for k in self.primary_keys)
+                try_to_allocate = True
                 if self.segmentation_obj.allocate(context, **segment):
                     LOG.debug('%(type)s segment allocate from pool success '
                               'with %(segment)s ', {'type': network_type,
                                                     'segment': segment})
                     return alloc
-                raise db_exc.RetryRequest(
-                    exceptions.NoNetworkFoundInMaximumAllowedAttempts())
+
+        if try_to_allocate:
+            raise db_exc.RetryRequest(
+                exceptions.NoNetworkFoundInMaximumAllowedAttempts())
 
     @db_api.retry_db_errors
     def _delete_expired_default_network_segment_ranges(self):

neutron/tests/functional/objects/plugins/ml2/test_base.py

@@ -0,0 +1,90 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import itertools
+
+from neutron_lib import context
+
+from neutron.tests.unit import testlib_api
+
+
+class _SegmentAllocation(testlib_api.SqlTestCase):
+
+    PHYSNETS = ('phys1', 'phys2')
+    NUM_SEGIDS = 10
+    segment_allocation_class = None
+
+    def setUp(self):
+        if not self.segment_allocation_class:
+            self.skipTest('No allocation class defined')
+        super().setUp()
+        self.context = context.Context(user_id='usier_id',
+                                       tenant_id='tenant_id')
+        self.segid_field = (
+            self.segment_allocation_class.get_segmentation_id().name)
+        self.is_vlan = ('physical_network' in
+                        self.segment_allocation_class.db_model.primary_keys())
+        pk_columns = self.segment_allocation_class.db_model.__table__.\
+            primary_key.columns
+        self.primary_keys = {col.name for col in pk_columns}
+        self.segments = None
+
+    def _create_segments(self, num_segids, physnets, allocated=False):
+
+        if self.is_vlan:
+            self.segments = list(itertools.product(physnets,
+                                                   range(1, num_segids + 1)))
+            kwargs_list = [
+                {'physical_network': physnet,
+                 self.segid_field: segid,
+                 'allocated': allocated} for physnet, segid in self.segments]
+        else:
+            self.segments = list(range(1, num_segids + 1))
+            kwargs_list = [{self.segid_field: segid,
+                            'allocated': allocated} for segid in self.segments]
+
+        for kwargs in kwargs_list:
+            self.segment_allocation_class(self.context, **kwargs).create()
+
+        self.assertTrue(
+            len(kwargs_list),
+            len(self.segment_allocation_class.get_objects(self.context)))
+
+    def test_get_random_unallocated_segment_and_allocate(self):
+        m_get = self.segment_allocation_class.get_random_unallocated_segment
+        m_alloc = self.segment_allocation_class.allocate
+        self._create_segments(self.NUM_SEGIDS, self.PHYSNETS)
+        for _ in range(len(self.segments)):
+            unalloc = m_get(self.context)
+            segment = dict((k, unalloc[k]) for k in self.primary_keys)
+            m_alloc(self.context, **segment)
+            if self.is_vlan:
+                self.segments.remove((unalloc['physical_network'],
+                                      unalloc.segmentation_id))
+            else:
+                self.segments.remove(unalloc.segmentation_id)
+
+        self.assertEqual(0, len(self.segments))
+        self.assertIsNone(m_get(self.context))
+
+
+class _SegmentAllocationMySQL(_SegmentAllocation,
+                              testlib_api.MySQLTestCaseMixin):
+    pass
+
+
+class _SegmentAllocationPostgreSQL(_SegmentAllocation,
+                                   testlib_api.PostgreSQLTestCaseMixin):
+    pass

neutron/tests/functional/objects/plugins/ml2/test_geneveallocation.py

@@ -0,0 +1,26 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron.objects.plugins.ml2 import geneveallocation
+from neutron.tests.functional.objects.plugins.ml2 import test_base
+
+
+class TestGeneveSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
+    segment_allocation_class = geneveallocation.GeneveAllocation
+
+
+class TestGeneveSegmentAllocationPostgreSQL(
+        test_base._SegmentAllocationPostgreSQL):
+    segment_allocation_class = geneveallocation.GeneveAllocation

neutron/tests/functional/objects/plugins/ml2/test_greallocation.py

@@ -0,0 +1,26 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron.objects.plugins.ml2 import greallocation
+from neutron.tests.functional.objects.plugins.ml2 import test_base
+
+
+class TestGreSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
+    segment_allocation_class = greallocation.GreAllocation
+
+
+class TestGreSegmentAllocationPostgreSQL(
+        test_base._SegmentAllocationPostgreSQL):
+    segment_allocation_class = greallocation.GreAllocation

neutron/tests/functional/objects/plugins/ml2/test_vlanallocation.py

@@ -0,0 +1,26 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron.objects.plugins.ml2 import vlanallocation
+from neutron.tests.functional.objects.plugins.ml2 import test_base
+
+
+class TestVlanSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
+    segment_allocation_class = vlanallocation.VlanAllocation
+
+
+class TestVlanSegmentAllocationPostgreSQL(
+        test_base._SegmentAllocationPostgreSQL):
+    segment_allocation_class = vlanallocation.VlanAllocation

neutron/tests/functional/objects/plugins/ml2/test_vxlanallocation.py

@@ -0,0 +1,26 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron.objects.plugins.ml2 import vxlanallocation
+from neutron.tests.functional.objects.plugins.ml2 import test_base
+
+
+class TestVxlanSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
+    segment_allocation_class = vxlanallocation.VxlanAllocation
+
+
+class TestVxlanSegmentAllocationPostgreSQL(
+        test_base._SegmentAllocationPostgreSQL):
+    segment_allocation_class = vxlanallocation.VxlanAllocation

neutron/tests/unit/objects/plugins/ml2/test_base.py

@@ -16,19 +16,19 @@
 
 class SegmentAllocationDbObjTestCase(object):
 
-    def test_get_unallocated_segments(self):
-        self.assertEqual(
-            [], self._test_class.get_unallocated_segments(self.context))
+    def test_get_random_unallocated_segment(self):
+        self.assertIsNone(
+            self._test_class.get_random_unallocated_segment(self.context))
 
         obj = self.objs[0]
         obj.allocated = True
         obj.create()
-        self.assertEqual(
-            [], self._test_class.get_unallocated_segments(self.context))
+        self.assertIsNone(
+            self._test_class.get_random_unallocated_segment(self.context))
 
         obj = self.objs[1]
         obj.allocated = False
         obj.create()
-        allocations = self._test_class.get_unallocated_segments(self.context)
-        self.assertEqual(1, len(allocations))
-        self.assertEqual(obj.segmentation_id, allocations[0].segmentation_id)
+        allocations = self._test_class.get_random_unallocated_segment(
+            self.context)
+        self.assertEqual(obj.segmentation_id, allocations.segmentation_id)

neutron/tests/unit/plugins/ml2/drivers/test_type_vlan.py

@@ -348,7 +348,7 @@ class VlanTypeAllocationTest(testlib_api.SqlTestCase):
         # for PROVIDER_NET.
         self.assertEqual(
                 {'network_type': 'vlan', 'physical_network': PROVIDER_NET,
-                 'segmentation_id': p_const.MIN_VLAN_TAG, 'mtu': 1500},
+                 'segmentation_id': mock.ANY, 'mtu': 1500},
                 driver.allocate_tenant_segment(ctx))