Update the processing of assigned addresses when assigning addresses

1.It is best not to use 'netaddr.IPSet.add',
  because _compact_single_network in 'IPSet.add' is quite time consuming

2.When the current address pool does not have enough addresses,
  all addresses are allocated from the current pool,
  and allocations are continued from the next address pool
  until all addresses are assigned.

Change-Id: I804a95fdaa3552c785e85ffab7b8ac849c634a87
Closes-Bug: #1813253
(cherry picked from commit 1746d7e0e6)

neutron/ipam/drivers/neutrondb_ipam/driver.py

@@ -159,9 +159,16 @@ class NeutronDbSubnet(ipam_base.Subnet):
 
     def _generate_ips(self, context, prefer_next=False, num_addresses=1):
         """Generate a set of IPs from the set of available addresses."""
-        ip_allocations = netaddr.IPSet()
-        for ipallocation in self.subnet_manager.list_allocations(context):
-            ip_allocations.add(ipallocation.ip_address)
+        allocated_ips = []
+        requested_num_addresses = num_addresses
+
+        allocations = self.subnet_manager.list_allocations(context)
+        # It is better not to use 'netaddr.IPSet.add',
+        # because _compact_single_network in 'IPSet.add'
+        # is quite time consuming.
+        ip_allocations = netaddr.IPSet(
+            [netaddr.IPAddress(allocation.ip_address)
+             for allocation in allocations])
 
         for ip_pool in self.subnet_manager.list_pools(context):
             ip_set = netaddr.IPSet()
@@ -170,34 +177,53 @@ class NeutronDbSubnet(ipam_base.Subnet):
             if av_set.size == 0:
                 continue
 
-            if av_set.size < num_addresses:
-                # Not enough addresses in pool to perform validation
-                # TODO(njohnston): How to handle when there are enough IPs but
-                # not enough in a single pool to satisfy the request?
-                continue
+            if av_set.size < requested_num_addresses:
+                # All addresses of the address pool are allocated
+                # for the first time and the remaining addresses
+                # will be allocated in the next address pools.
+                allocated_num_addresses = av_set.size
+            else:
+                # All expected addresses can be assigned in this loop.
+                allocated_num_addresses = requested_num_addresses
 
             if prefer_next:
                 allocated_ip_pool = list(itertools.islice(av_set,
-                    num_addresses))
-                return [str(allocated_ip)
-                        for allocated_ip in allocated_ip_pool]
+                    allocated_num_addresses))
+                allocated_ips.extend([str(allocated_ip)
+                                      for allocated_ip in allocated_ip_pool])
+
+                requested_num_addresses -= allocated_num_addresses
+                if requested_num_addresses:
+                    # More addresses need to be allocated in the next loop.
+                    continue
+                return allocated_ips
 
             window = min(av_set.size, MAX_WIN)
 
             # NOTE(gryf): If there is more than one address, make the window
             # bigger, so that are chances to fulfill demanded amount of IPs.
-            if num_addresses > 1:
-                window = min(av_set.size, num_addresses * MULTIPLIER,
+            if allocated_num_addresses > 1:
+                window = min(av_set.size,
+                             allocated_num_addresses * MULTIPLIER,
                              MAX_WIN_MULTI)
 
-            if window < num_addresses:
+            if window < allocated_num_addresses:
                 continue
             else:
                 # Maximize randomness by using the random module's built in
                 # sampling function
                 av_ips = list(itertools.islice(av_set, 0, window))
-                allocated_ip_pool = random.sample(av_ips, num_addresses)
-            return [str(allocated_ip) for allocated_ip in allocated_ip_pool]
+                allocated_ip_pool = random.sample(av_ips,
+                                                  allocated_num_addresses)
+                allocated_ips.extend([str(allocated_ip)
+                                      for allocated_ip in allocated_ip_pool])
+
+            requested_num_addresses -= allocated_num_addresses
+            if requested_num_addresses:
+                # More addresses need to be allocated in the next loop.
+                continue
+
+            return allocated_ips
 
         raise ipam_exc.IpAddressGenerationFailure(
                   subnet_id=self.subnet_manager.neutron_id)

neutron/tests/unit/ipam/drivers/neutrondb_ipam/test_driver.py

@@ -381,6 +381,31 @@ class TestNeutronDbIpamSubnet(testlib_api.SqlTestCase,
                           ipam_subnet.bulk_allocate,
                           ipam_req.BulkAddressRequest(target_ip_count))
 
+    def test_bulk_allocate_multiple_address_pools(self):
+        target_ip_count = 10
+        # 11 addresses available
+        allocation_pools = [{'start': '192.168.0.5', 'end': '192.168.0.9'},
+                            {'start': '192.168.0.15', 'end': '192.168.0.20'}]
+        ipam_subnet = self._create_and_allocate_ipam_subnet(
+            '192.168.0.0/24', allocation_pools=allocation_pools,
+            ip_version=constants.IP_VERSION_4)[0]
+        ip_addresses = ipam_subnet.bulk_allocate(
+            ipam_req.BulkAddressRequest(target_ip_count))
+        self.assertEqual(target_ip_count, len(ip_addresses))
+        self.assertRaises(ipam_exc.IpAddressGenerationFailure,
+                          ipam_subnet.bulk_allocate,
+                          ipam_req.BulkAddressRequest(2))
+
+    def test_prefernext_allocate_multiple_address_pools(self):
+        ipam_subnet = self._create_and_allocate_ipam_subnet(
+            '192.168.0.0/30', ip_version=constants.IP_VERSION_4)[0]
+
+        ipam_subnet.allocate(ipam_req.PreferNextAddressRequest())
+        # The second address generation request on a /30 for v4 net must fail
+        self.assertRaises(ipam_exc.IpAddressGenerationFailure,
+                          ipam_subnet.allocate,
+                          ipam_req.PreferNextAddressRequest)
+
     def _test_deallocate_address(self, cidr, ip_version):
         ipam_subnet = self._create_and_allocate_ipam_subnet(
             cidr, ip_version=ip_version)[0]