From 06e38be42e3996eb8c142c657aa80c825e767f6d Mon Sep 17 00:00:00 2001 From: Nate Johnston Date: Wed, 15 Aug 2018 13:49:19 -0400 Subject: [PATCH] Add bulk IP address assignment to ipam driver Create a method for bulk assignment of IP addresses within the ipam driver, to support bulk creation of ports. This also changes the logic for how the window of available IP addresses to assign from is calculated within the neutrondb IPAM driver. The Python random module is used to produce a statistically sampled set of IP addresses out of the set of available IPs; this will facilitate collission avoidance. When requesting multiple IP addresses the selection window sized is increased significantly to ensure a larger number of available IPs, but caps are placed on the amount to make sure we do not transgress system limits when building pools of IPv6 addresses. Change-Id: Iad8088eaa261b07153fa358ae34b9a2442bc2a3e Implements: blueprint speed-up-neutron-bulk-creation --- neutron/ipam/drivers/neutrondb_ipam/driver.py | 65 +++++++++++++++---- neutron/ipam/requests.py | 15 +++++ .../drivers/neutrondb_ipam/test_driver.py | 22 +++++++ 3 files changed, 91 insertions(+), 11 deletions(-) diff --git a/neutron/ipam/drivers/neutrondb_ipam/driver.py b/neutron/ipam/drivers/neutrondb_ipam/driver.py index f59ca14e3d1..996ba7ef783 100644 --- a/neutron/ipam/drivers/neutrondb_ipam/driver.py +++ b/neutron/ipam/drivers/neutrondb_ipam/driver.py @@ -33,6 +33,9 @@ from neutron.ipam import utils as ipam_utils LOG = log.getLogger(__name__) +MAX_WIN = 1000 +MULTIPLIER = 100 +MAX_WIN_MULTI = MAX_WIN * MULTIPLIER class NeutronDbSubnet(ipam_base.Subnet): @@ -152,6 +155,10 @@ class NeutronDbSubnet(ipam_base.Subnet): def _generate_ip(self, context, prefer_next=False): """Generate an IP address from the set of available addresses.""" + return self._generate_ips(context, prefer_next)[0] + + def _generate_ips(self, context, prefer_next=False, num_addresses=1): + """Generate a set of IPs from the set of available addresses.""" ip_allocations = netaddr.IPSet() for ipallocation in self.subnet_manager.list_allocations(context): ip_allocations.add(ipallocation.ip_address) @@ -163,16 +170,34 @@ class NeutronDbSubnet(ipam_base.Subnet): if av_set.size == 0: continue + if av_set.size < num_addresses: + # Not enough addresses in pool to perform validation + # TODO(njohnston): How to handle when there are enough IPs but + # not enough in a single pool to satisfy the request? + continue + if prefer_next: - window = 1 + allocated_ip_pool = list(itertools.islice(av_set, + num_addresses)) + return [str(allocated_ip) + for allocated_ip in allocated_ip_pool] + + window = min(av_set.size, MAX_WIN) + + # NOTE(gryf): If there is more than one address, make the window + # bigger, so that are chances to fulfill demanded amount of IPs. + if num_addresses > 1: + window = min(av_set.size, num_addresses * MULTIPLIER, + MAX_WIN_MULTI) + + if window < num_addresses: + continue else: - # Compute a value for the selection window - window = min(av_set.size, 30) - ip_index = random.randint(1, window) - candidate_ips = list(itertools.islice(av_set, ip_index)) - allocated_ip = candidate_ips[ - random.randint(0, len(candidate_ips) - 1)] - return str(allocated_ip), ip_pool.id + # Maximize randomness by using the random module's built in + # sampling function + av_ips = list(itertools.islice(av_set, 0, window)) + allocated_ip_pool = random.sample(av_ips, num_addresses) + return [str(allocated_ip) for allocated_ip in allocated_ip_pool] raise ipam_exc.IpAddressGenerationFailure( subnet_id=self.subnet_manager.neutron_id) @@ -182,7 +207,6 @@ class NeutronDbSubnet(ipam_base.Subnet): # running transaction, which is started on create_port or upper level. # To be able to do rollback/retry actions correctly ipam driver # should not create new nested transaction blocks. - all_pool_id = None # NOTE(salv-orlando): It would probably better to have a simpler # model for address requests and just check whether there is a # specific IP address specified in address_request @@ -194,8 +218,7 @@ class NeutronDbSubnet(ipam_base.Subnet): else: prefer_next = isinstance(address_request, ipam_req.PreferNextAddressRequest) - ip_address, all_pool_id = self._generate_ip(self._context, - prefer_next) + ip_address = self._generate_ip(self._context, prefer_next) # Create IP allocation request object # The only defined status at this stage is 'ALLOCATED'. @@ -215,6 +238,26 @@ class NeutronDbSubnet(ipam_base.Subnet): subnet_id=self.subnet_manager.neutron_id) return ip_address + def bulk_allocate(self, address_request): + # The signature of this function differs from allocate only in that it + # returns a list of addresses, as opposed to a single address. + if not isinstance(address_request, ipam_req.BulkAddressRequest): + return [self.allocate(address_request)] + num_addrs = address_request.num_addresses + allocated_ip_pool = self._generate_ips(self._context, + False, + num_addrs) + # Create IP allocation request objects + try: + with self._context.session.begin(subtransactions=True): + for ip_address in allocated_ip_pool: + self.subnet_manager.create_allocation(self._context, + ip_address) + except db_exc.DBReferenceError: + raise n_exc.SubnetNotFound( + subnet_id=self.subnet_manager.neutron_id) + return allocated_ip_pool + def deallocate(self, address): # This is almost a no-op because the Neutron DB IPAM driver does not # delete IPAllocation objects at every deallocation. The only diff --git a/neutron/ipam/requests.py b/neutron/ipam/requests.py index 2393e82a331..a65b19ce260 100644 --- a/neutron/ipam/requests.py +++ b/neutron/ipam/requests.py @@ -205,6 +205,21 @@ class SpecificAddressRequest(AddressRequest): return self._address +class BulkAddressRequest(AddressRequest): + """For requesting a batch of available addresses from IPAM""" + def __init__(self, num_addresses): + """Initialize BulkAddressRequest + :param num_addresses: The quantity of IP addresses being requested + :type num_addresses: int + """ + super(BulkAddressRequest, self).__init__() + self._num_addresses = num_addresses + + @property + def num_addresses(self): + return self._num_addresses + + class AnyAddressRequest(AddressRequest): """Used to request any available address from the pool.""" diff --git a/neutron/tests/unit/ipam/drivers/neutrondb_ipam/test_driver.py b/neutron/tests/unit/ipam/drivers/neutrondb_ipam/test_driver.py index c79146b8d38..8b37b4618f9 100644 --- a/neutron/tests/unit/ipam/drivers/neutrondb_ipam/test_driver.py +++ b/neutron/tests/unit/ipam/drivers/neutrondb_ipam/test_driver.py @@ -361,6 +361,28 @@ class TestNeutronDbIpamSubnet(testlib_api.SqlTestCase, ipam_subnet.allocate, ipam_req.AnyAddressRequest) + def test_bulk_allocate_v4_address(self): + target_ip_count = 10 + ipam_subnet = self._create_and_allocate_ipam_subnet( + '192.168.0.0/28', ip_version=constants.IP_VERSION_4)[0] + ip_addresses = ipam_subnet.bulk_allocate( + ipam_req.BulkAddressRequest(target_ip_count)) + self.assertEqual(target_ip_count, len(ip_addresses)) + self.assertRaises(ipam_exc.IpAddressGenerationFailure, + ipam_subnet.bulk_allocate, + ipam_req.BulkAddressRequest(target_ip_count)) + + def test_bulk_allocate_v6_address(self): + target_ip_count = 10 + ipam_subnet = self._create_and_allocate_ipam_subnet( + 'fd00::/124', ip_version=constants.IP_VERSION_6)[0] + ip_addresses = ipam_subnet.bulk_allocate( + ipam_req.BulkAddressRequest(target_ip_count)) + self.assertEqual(target_ip_count, len(ip_addresses)) + self.assertRaises(ipam_exc.IpAddressGenerationFailure, + ipam_subnet.bulk_allocate, + ipam_req.BulkAddressRequest(target_ip_count)) + def _test_deallocate_address(self, cidr, ip_version): ipam_subnet = self._create_and_allocate_ipam_subnet( cidr, ip_version=ip_version)[0]