Add bulk IP address assignment to ipam driver

Create a method for bulk assignment of IP addresses within the ipam
driver, to support bulk creation of ports.

This also changes the logic for how the window of available IP addresses
to assign from is calculated within the neutrondb IPAM driver.  The
Python random module is used to produce a statistically sampled set of
IP addresses out of the set of available IPs; this will facilitate
collission avoidance.  When requesting multiple IP addresses the
selection window sized is increased significantly to ensure a larger
number of available IPs, but caps are placed on the amount to make sure
we do not transgress system limits when building pools of IPv6
addresses.

Change-Id: Iad8088eaa261b07153fa358ae34b9a2442bc2a3e
Implements: blueprint speed-up-neutron-bulk-creation

neutron/ipam/drivers/neutrondb_ipam/driver.py

@@ -33,6 +33,9 @@ from neutron.ipam import utils as ipam_utils
 
 
 LOG = log.getLogger(__name__)
+MAX_WIN = 1000
+MULTIPLIER = 100
+MAX_WIN_MULTI = MAX_WIN * MULTIPLIER
 
 
 class NeutronDbSubnet(ipam_base.Subnet):
@@ -152,6 +155,10 @@ class NeutronDbSubnet(ipam_base.Subnet):
 
     def _generate_ip(self, context, prefer_next=False):
         """Generate an IP address from the set of available addresses."""
+        return self._generate_ips(context, prefer_next)[0]
+
+    def _generate_ips(self, context, prefer_next=False, num_addresses=1):
+        """Generate a set of IPs from the set of available addresses."""
         ip_allocations = netaddr.IPSet()
         for ipallocation in self.subnet_manager.list_allocations(context):
             ip_allocations.add(ipallocation.ip_address)
@@ -163,16 +170,34 @@ class NeutronDbSubnet(ipam_base.Subnet):
             if av_set.size == 0:
                 continue
 
+            if av_set.size < num_addresses:
+                # Not enough addresses in pool to perform validation
+                # TODO(njohnston): How to handle when there are enough IPs but
+                # not enough in a single pool to satisfy the request?
+                continue
+
             if prefer_next:
-                window = 1
+                allocated_ip_pool = list(itertools.islice(av_set,
+                    num_addresses))
+                return [str(allocated_ip)
+                        for allocated_ip in allocated_ip_pool]
+
+            window = min(av_set.size, MAX_WIN)
+
+            # NOTE(gryf): If there is more than one address, make the window
+            # bigger, so that are chances to fulfill demanded amount of IPs.
+            if num_addresses > 1:
+                window = min(av_set.size, num_addresses * MULTIPLIER,
+                             MAX_WIN_MULTI)
+
+            if window < num_addresses:
+                continue
             else:
-                # Compute a value for the selection window
+                # Maximize randomness by using the random module's built in
-                window = min(av_set.size, 30)
+                # sampling function
-            ip_index = random.randint(1, window)
+                av_ips = list(itertools.islice(av_set, 0, window))
-            candidate_ips = list(itertools.islice(av_set, ip_index))
+                allocated_ip_pool = random.sample(av_ips, num_addresses)
-            allocated_ip = candidate_ips[
+            return [str(allocated_ip) for allocated_ip in allocated_ip_pool]
-                random.randint(0, len(candidate_ips) - 1)]
-            return str(allocated_ip), ip_pool.id
 
         raise ipam_exc.IpAddressGenerationFailure(
                   subnet_id=self.subnet_manager.neutron_id)
@@ -182,7 +207,6 @@ class NeutronDbSubnet(ipam_base.Subnet):
         # running transaction, which is started on create_port or upper level.
         # To be able to do rollback/retry actions correctly ipam driver
         # should not create new nested transaction blocks.
-        all_pool_id = None
         # NOTE(salv-orlando): It would probably better to have a simpler
         # model for address requests and just check whether there is a
         # specific IP address specified in address_request
@@ -194,8 +218,7 @@ class NeutronDbSubnet(ipam_base.Subnet):
         else:
             prefer_next = isinstance(address_request,
                                      ipam_req.PreferNextAddressRequest)
-            ip_address, all_pool_id = self._generate_ip(self._context,
+            ip_address = self._generate_ip(self._context, prefer_next)
-                                                        prefer_next)
 
         # Create IP allocation request object
         # The only defined status at this stage is 'ALLOCATED'.
@@ -215,6 +238,26 @@ class NeutronDbSubnet(ipam_base.Subnet):
                 subnet_id=self.subnet_manager.neutron_id)
         return ip_address
 
+    def bulk_allocate(self, address_request):
+        # The signature of this function differs from allocate only in that it
+        # returns a list of addresses, as opposed to a single address.
+        if not isinstance(address_request, ipam_req.BulkAddressRequest):
+            return [self.allocate(address_request)]
+        num_addrs = address_request.num_addresses
+        allocated_ip_pool = self._generate_ips(self._context,
+                                               False,
+                                               num_addrs)
+        # Create IP allocation request objects
+        try:
+            with self._context.session.begin(subtransactions=True):
+                for ip_address in allocated_ip_pool:
+                    self.subnet_manager.create_allocation(self._context,
+                                                          ip_address)
+        except db_exc.DBReferenceError:
+            raise n_exc.SubnetNotFound(
+                subnet_id=self.subnet_manager.neutron_id)
+        return allocated_ip_pool
+
     def deallocate(self, address):
         # This is almost a no-op because the Neutron DB IPAM driver does not
         # delete IPAllocation objects at every deallocation. The only

neutron/ipam/requests.py

@@ -205,6 +205,21 @@ class SpecificAddressRequest(AddressRequest):
         return self._address
 
 
+class BulkAddressRequest(AddressRequest):
+    """For requesting a batch of available addresses from IPAM"""
+    def __init__(self, num_addresses):
+        """Initialize BulkAddressRequest
+        :param num_addresses: The quantity of IP addresses being requested
+        :type num_addresses: int
+        """
+        super(BulkAddressRequest, self).__init__()
+        self._num_addresses = num_addresses
+
+    @property
+    def num_addresses(self):
+        return self._num_addresses
+
+
 class AnyAddressRequest(AddressRequest):
     """Used to request any available address from the pool."""
 

neutron/tests/unit/ipam/drivers/neutrondb_ipam/test_driver.py

@@ -361,6 +361,28 @@ class TestNeutronDbIpamSubnet(testlib_api.SqlTestCase,
                           ipam_subnet.allocate,
                           ipam_req.AnyAddressRequest)
 
+    def test_bulk_allocate_v4_address(self):
+        target_ip_count = 10
+        ipam_subnet = self._create_and_allocate_ipam_subnet(
+            '192.168.0.0/28', ip_version=constants.IP_VERSION_4)[0]
+        ip_addresses = ipam_subnet.bulk_allocate(
+                ipam_req.BulkAddressRequest(target_ip_count))
+        self.assertEqual(target_ip_count, len(ip_addresses))
+        self.assertRaises(ipam_exc.IpAddressGenerationFailure,
+                          ipam_subnet.bulk_allocate,
+                          ipam_req.BulkAddressRequest(target_ip_count))
+
+    def test_bulk_allocate_v6_address(self):
+        target_ip_count = 10
+        ipam_subnet = self._create_and_allocate_ipam_subnet(
+                'fd00::/124', ip_version=constants.IP_VERSION_6)[0]
+        ip_addresses = ipam_subnet.bulk_allocate(
+                ipam_req.BulkAddressRequest(target_ip_count))
+        self.assertEqual(target_ip_count, len(ip_addresses))
+        self.assertRaises(ipam_exc.IpAddressGenerationFailure,
+                          ipam_subnet.bulk_allocate,
+                          ipam_req.BulkAddressRequest(target_ip_count))
+
     def _test_deallocate_address(self, cidr, ip_version):
         ipam_subnet = self._create_and_allocate_ipam_subnet(
             cidr, ip_version=ip_version)[0]