Merge "Randomize segmentation ID assignation" into stable/train

neutron/common/utils.py

@@ -48,6 +48,10 @@ from oslo_utils import timeutils
 from oslo_utils import uuidutils
 from osprofiler import profiler
 import pkg_resources
+from sqlalchemy.dialects.mysql import dialect as mysql_dialect
+from sqlalchemy.dialects.postgresql import dialect as postgresql_dialect
+from sqlalchemy.dialects.sqlite import dialect as sqlite_dialect
+from sqlalchemy.sql.expression import func as sql_func
 
 import neutron
 from neutron._i18n import _
@@ -1016,3 +1020,16 @@ def skip_exceptions(exceptions):
                         ctx.reraise = False
         return wrapper
     return decorator
+
+
+def get_sql_random_method(sql_dialect_name):
+    """Return the SQL random method supported depending on the dialect."""
+    # NOTE(ralonsoh): this method is a good candidate to be implemented in
+    # oslo.db.
+    # https://www.postgresql.org/docs/8.2/functions-math.html
+    # https://www.sqlite.org/c3ref/randomness.html
+    if sql_dialect_name in (postgresql_dialect.name, sqlite_dialect.name):
+        return sql_func.random
+    # https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html
+    elif sql_dialect_name == mysql_dialect.name:
+        return sql_func.rand

neutron/objects/plugins/ml2/base.py

@@ -17,7 +17,7 @@ import abc
 import netaddr
 import six
 
-from neutron.common import _constants as common_constants
+from neutron.common import utils as n_utils
 from neutron.objects import base
 
 
@@ -44,14 +44,18 @@ class EndpointBase(base.NeutronDbObject):
 class SegmentAllocation(object):
 
     @classmethod
-    def get_unallocated_segments(cls, context, **filters):
+    def get_random_unallocated_segment(cls, context, **filters):
         with cls.db_context_reader(context):
             columns = set(dict(cls.db_model.__table__.columns))
             model_filters = dict((k, filters[k])
                                  for k in columns & set(filters.keys()))
             query = context.session.query(cls.db_model).filter_by(
                 allocated=False, **model_filters)
-            return query.limit(common_constants.IDPOOL_SELECT_SIZE).all()
+            rand_func = n_utils.get_sql_random_method(
+                context.session.bind.dialect.name)
+            if rand_func:
+                query = query.order_by(rand_func())
+            return query.first()
 
     @classmethod
     def allocate(cls, context, **segment):

neutron/plugins/ml2/drivers/helpers.py

@@ -137,17 +137,23 @@ class SegmentTypeDriver(BaseTypeDriver):
                     self.model_segmentation_id, **filters)]
         else:
             calls = [functools.partial(
-                self.segmentation_obj.get_unallocated_segments,
+                self.segmentation_obj.get_random_unallocated_segment,
                 context, **filters)]
 
+        try_to_allocate = False
         for call in calls:
             allocations = call()
+            if not isinstance(allocations, list):
+                allocations = [allocations] if allocations else []
             for alloc in allocations:
                 segment = dict((k, alloc[k]) for k in self.primary_keys)
+                try_to_allocate = True
                 if self.segmentation_obj.allocate(context, **segment):
                     LOG.debug('%(type)s segment allocate from pool success '
                               'with %(segment)s ', {'type': network_type,
                                                     'segment': segment})
                     return alloc
-                raise db_exc.RetryRequest(
-                    exceptions.NoNetworkFoundInMaximumAllowedAttempts())
+
+        if try_to_allocate:
+            raise db_exc.RetryRequest(
+                exceptions.NoNetworkFoundInMaximumAllowedAttempts())

neutron/tests/functional/objects/plugins/ml2/test_base.py

@@ -0,0 +1,90 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import itertools
+
+from neutron_lib import context
+
+from neutron.tests.unit import testlib_api
+
+
+class _SegmentAllocation(testlib_api.SqlTestCase):
+
+    PHYSNETS = ('phys1', 'phys2')
+    NUM_SEGIDS = 10
+    segment_allocation_class = None
+
+    def setUp(self):
+        if not self.segment_allocation_class:
+            self.skipTest('No allocation class defined')
+        super(_SegmentAllocation, self).setUp()
+        self.context = context.Context(user_id='usier_id',
+                                       tenant_id='tenant_id')
+        self.segid_field = (
+            self.segment_allocation_class.get_segmentation_id().name)
+        self.is_vlan = ('physical_network' in
+                        self.segment_allocation_class.db_model.primary_keys())
+        pk_columns = self.segment_allocation_class.db_model.__table__.\
+            primary_key.columns
+        self.primary_keys = {col.name for col in pk_columns}
+        self.segments = None
+
+    def _create_segments(self, num_segids, physnets, allocated=False):
+
+        if self.is_vlan:
+            self.segments = list(itertools.product(physnets,
+                                                   range(1, num_segids + 1)))
+            kwargs_list = [
+                {'physical_network': physnet,
+                 self.segid_field: segid,
+                 'allocated': allocated} for physnet, segid in self.segments]
+        else:
+            self.segments = list(range(1, num_segids + 1))
+            kwargs_list = [{self.segid_field: segid,
+                            'allocated': allocated} for segid in self.segments]
+
+        for kwargs in kwargs_list:
+            self.segment_allocation_class(self.context, **kwargs).create()
+
+        self.assertTrue(
+            len(kwargs_list),
+            len(self.segment_allocation_class.get_objects(self.context)))
+
+    def test_get_random_unallocated_segment_and_allocate(self):
+        m_get = self.segment_allocation_class.get_random_unallocated_segment
+        m_alloc = self.segment_allocation_class.allocate
+        self._create_segments(self.NUM_SEGIDS, self.PHYSNETS)
+        for _ in range(len(self.segments)):
+            unalloc = m_get(self.context)
+            segment = dict((k, unalloc[k]) for k in self.primary_keys)
+            m_alloc(self.context, **segment)
+            if self.is_vlan:
+                self.segments.remove((unalloc['physical_network'],
+                                      unalloc.segmentation_id))
+            else:
+                self.segments.remove(unalloc.segmentation_id)
+
+        self.assertEqual(0, len(self.segments))
+        self.assertIsNone(m_get(self.context))
+
+
+class _SegmentAllocationMySQL(_SegmentAllocation,
+                              testlib_api.MySQLTestCaseMixin):
+    pass
+
+
+class _SegmentAllocationPostgreSQL(_SegmentAllocation,
+                                   testlib_api.PostgreSQLTestCaseMixin):
+    pass

neutron/tests/functional/objects/plugins/ml2/test_geneveallocation.py

@@ -0,0 +1,26 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron.objects.plugins.ml2 import geneveallocation
+from neutron.tests.functional.objects.plugins.ml2 import test_base
+
+
+class TestGeneveSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
+    segment_allocation_class = geneveallocation.GeneveAllocation
+
+
+class TestGeneveSegmentAllocationPostgreSQL(
+        test_base._SegmentAllocationPostgreSQL):
+    segment_allocation_class = geneveallocation.GeneveAllocation

neutron/tests/functional/objects/plugins/ml2/test_greallocation.py

@@ -0,0 +1,26 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron.objects.plugins.ml2 import greallocation
+from neutron.tests.functional.objects.plugins.ml2 import test_base
+
+
+class TestGreSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
+    segment_allocation_class = greallocation.GreAllocation
+
+
+class TestGreSegmentAllocationPostgreSQL(
+        test_base._SegmentAllocationPostgreSQL):
+    segment_allocation_class = greallocation.GreAllocation

neutron/tests/functional/objects/plugins/ml2/test_vlanallocation.py

@@ -0,0 +1,26 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron.objects.plugins.ml2 import vlanallocation
+from neutron.tests.functional.objects.plugins.ml2 import test_base
+
+
+class TestVlanSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
+    segment_allocation_class = vlanallocation.VlanAllocation
+
+
+class TestVlanSegmentAllocationPostgreSQL(
+        test_base._SegmentAllocationPostgreSQL):
+    segment_allocation_class = vlanallocation.VlanAllocation

neutron/tests/functional/objects/plugins/ml2/test_vxlanallocation.py

@@ -0,0 +1,26 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron.objects.plugins.ml2 import vxlanallocation
+from neutron.tests.functional.objects.plugins.ml2 import test_base
+
+
+class TestVxlanSegmentAllocationMySQL(test_base._SegmentAllocationMySQL):
+    segment_allocation_class = vxlanallocation.VxlanAllocation
+
+
+class TestVxlanSegmentAllocationPostgreSQL(
+        test_base._SegmentAllocationPostgreSQL):
+    segment_allocation_class = vxlanallocation.VxlanAllocation

neutron/tests/unit/objects/plugins/ml2/test_base.py

@@ -16,19 +16,19 @@
 
 class SegmentAllocationDbObjTestCase(object):
 
-    def test_get_unallocated_segments(self):
-        self.assertEqual(
-            [], self._test_class.get_unallocated_segments(self.context))
+    def test_get_random_unallocated_segment(self):
+        self.assertIsNone(
+            self._test_class.get_random_unallocated_segment(self.context))
 
         obj = self.objs[0]
         obj.allocated = True
         obj.create()
-        self.assertEqual(
-            [], self._test_class.get_unallocated_segments(self.context))
+        self.assertIsNone(
+            self._test_class.get_random_unallocated_segment(self.context))
 
         obj = self.objs[1]
         obj.allocated = False
         obj.create()
-        allocations = self._test_class.get_unallocated_segments(self.context)
-        self.assertEqual(1, len(allocations))
-        self.assertEqual(obj.segmentation_id, allocations[0].segmentation_id)
+        allocations = self._test_class.get_random_unallocated_segment(
+            self.context)
+        self.assertEqual(obj.segmentation_id, allocations.segmentation_id)