From c8fec8bb3c7918fd0c0136abe6fc1e482a6b28ee Mon Sep 17 00:00:00 2001 From: Nate Johnston Date: Wed, 15 Aug 2018 13:49:19 -0400 Subject: [PATCH] Add bulk IP address assignment to ipam driver Create a method for bulk assignment of IP addresses within the ipam driver, to support bulk creation of ports. This also changes the logic for how the window of available IP addresses to assign from is calculated within the neutrondb IPAM driver. The Python random module is used to produce a statistically sampled set of IP addresses out of the set of available IPs; this will facilitate collission avoidance. When requesting multiple IP addresses the selection window sized is increased significantly to ensure a larger number of available IPs, but caps are placed on the amount to make sure we do not transgress system limits when building pools of IPv6 addresses. Change-Id: Iad8088eaa261b07153fa358ae34b9a2442bc2a3e Implements: blueprint speed-up-neutron-bulk-creation (cherry picked from commit 06e38be42e3996eb8c142c657aa80c825e767f6d) --- neutron/ipam/drivers/neutrondb_ipam/driver.py | 65 +++++++++++++++---- neutron/ipam/requests.py | 15 +++++ .../drivers/neutrondb_ipam/test_driver.py | 22 +++++++ 3 files changed, 91 insertions(+), 11 deletions(-) diff --git a/neutron/ipam/drivers/neutrondb_ipam/driver.py b/neutron/ipam/drivers/neutrondb_ipam/driver.py index f59ca14e3d1..996ba7ef783 100644 --- a/neutron/ipam/drivers/neutrondb_ipam/driver.py +++ b/neutron/ipam/drivers/neutrondb_ipam/driver.py @@ -33,6 +33,9 @@ from neutron.ipam import utils as ipam_utils LOG = log.getLogger(__name__) +MAX_WIN = 1000 +MULTIPLIER = 100 +MAX_WIN_MULTI = MAX_WIN * MULTIPLIER class NeutronDbSubnet(ipam_base.Subnet): @@ -152,6 +155,10 @@ class NeutronDbSubnet(ipam_base.Subnet): def _generate_ip(self, context, prefer_next=False): """Generate an IP address from the set of available addresses.""" + return self._generate_ips(context, prefer_next)[0] + + def _generate_ips(self, context, prefer_next=False, num_addresses=1): + """Generate a set of IPs from the set of available addresses.""" ip_allocations = netaddr.IPSet() for ipallocation in self.subnet_manager.list_allocations(context): ip_allocations.add(ipallocation.ip_address) @@ -163,16 +170,34 @@ class NeutronDbSubnet(ipam_base.Subnet): if av_set.size == 0: continue + if av_set.size < num_addresses: + # Not enough addresses in pool to perform validation + # TODO(njohnston): How to handle when there are enough IPs but + # not enough in a single pool to satisfy the request? + continue + if prefer_next: - window = 1 + allocated_ip_pool = list(itertools.islice(av_set, + num_addresses)) + return [str(allocated_ip) + for allocated_ip in allocated_ip_pool] + + window = min(av_set.size, MAX_WIN) + + # NOTE(gryf): If there is more than one address, make the window + # bigger, so that are chances to fulfill demanded amount of IPs. + if num_addresses > 1: + window = min(av_set.size, num_addresses * MULTIPLIER, + MAX_WIN_MULTI) + + if window < num_addresses: + continue else: - # Compute a value for the selection window - window = min(av_set.size, 30) - ip_index = random.randint(1, window) - candidate_ips = list(itertools.islice(av_set, ip_index)) - allocated_ip = candidate_ips[ - random.randint(0, len(candidate_ips) - 1)] - return str(allocated_ip), ip_pool.id + # Maximize randomness by using the random module's built in + # sampling function + av_ips = list(itertools.islice(av_set, 0, window)) + allocated_ip_pool = random.sample(av_ips, num_addresses) + return [str(allocated_ip) for allocated_ip in allocated_ip_pool] raise ipam_exc.IpAddressGenerationFailure( subnet_id=self.subnet_manager.neutron_id) @@ -182,7 +207,6 @@ class NeutronDbSubnet(ipam_base.Subnet): # running transaction, which is started on create_port or upper level. # To be able to do rollback/retry actions correctly ipam driver # should not create new nested transaction blocks. - all_pool_id = None # NOTE(salv-orlando): It would probably better to have a simpler # model for address requests and just check whether there is a # specific IP address specified in address_request @@ -194,8 +218,7 @@ class NeutronDbSubnet(ipam_base.Subnet): else: prefer_next = isinstance(address_request, ipam_req.PreferNextAddressRequest) - ip_address, all_pool_id = self._generate_ip(self._context, - prefer_next) + ip_address = self._generate_ip(self._context, prefer_next) # Create IP allocation request object # The only defined status at this stage is 'ALLOCATED'. @@ -215,6 +238,26 @@ class NeutronDbSubnet(ipam_base.Subnet): subnet_id=self.subnet_manager.neutron_id) return ip_address + def bulk_allocate(self, address_request): + # The signature of this function differs from allocate only in that it + # returns a list of addresses, as opposed to a single address. + if not isinstance(address_request, ipam_req.BulkAddressRequest): + return [self.allocate(address_request)] + num_addrs = address_request.num_addresses + allocated_ip_pool = self._generate_ips(self._context, + False, + num_addrs) + # Create IP allocation request objects + try: + with self._context.session.begin(subtransactions=True): + for ip_address in allocated_ip_pool: + self.subnet_manager.create_allocation(self._context, + ip_address) + except db_exc.DBReferenceError: + raise n_exc.SubnetNotFound( + subnet_id=self.subnet_manager.neutron_id) + return allocated_ip_pool + def deallocate(self, address): # This is almost a no-op because the Neutron DB IPAM driver does not # delete IPAllocation objects at every deallocation. The only diff --git a/neutron/ipam/requests.py b/neutron/ipam/requests.py index 95a4a67c708..302a375af34 100644 --- a/neutron/ipam/requests.py +++ b/neutron/ipam/requests.py @@ -202,6 +202,21 @@ class SpecificAddressRequest(AddressRequest): return self._address +class BulkAddressRequest(AddressRequest): + """For requesting a batch of available addresses from IPAM""" + def __init__(self, num_addresses): + """Initialize BulkAddressRequest + :param num_addresses: The quantity of IP addresses being requested + :type num_addresses: int + """ + super(BulkAddressRequest, self).__init__() + self._num_addresses = num_addresses + + @property + def num_addresses(self): + return self._num_addresses + + class AnyAddressRequest(AddressRequest): """Used to request any available address from the pool.""" diff --git a/neutron/tests/unit/ipam/drivers/neutrondb_ipam/test_driver.py b/neutron/tests/unit/ipam/drivers/neutrondb_ipam/test_driver.py index a607c9e5846..39eb6cc3975 100644 --- a/neutron/tests/unit/ipam/drivers/neutrondb_ipam/test_driver.py +++ b/neutron/tests/unit/ipam/drivers/neutrondb_ipam/test_driver.py @@ -359,6 +359,28 @@ class TestNeutronDbIpamSubnet(testlib_api.SqlTestCase, ipam_subnet.allocate, ipam_req.AnyAddressRequest) + def test_bulk_allocate_v4_address(self): + target_ip_count = 10 + ipam_subnet = self._create_and_allocate_ipam_subnet( + '192.168.0.0/28', ip_version=constants.IP_VERSION_4)[0] + ip_addresses = ipam_subnet.bulk_allocate( + ipam_req.BulkAddressRequest(target_ip_count)) + self.assertEqual(target_ip_count, len(ip_addresses)) + self.assertRaises(ipam_exc.IpAddressGenerationFailure, + ipam_subnet.bulk_allocate, + ipam_req.BulkAddressRequest(target_ip_count)) + + def test_bulk_allocate_v6_address(self): + target_ip_count = 10 + ipam_subnet = self._create_and_allocate_ipam_subnet( + 'fd00::/124', ip_version=constants.IP_VERSION_6)[0] + ip_addresses = ipam_subnet.bulk_allocate( + ipam_req.BulkAddressRequest(target_ip_count)) + self.assertEqual(target_ip_count, len(ip_addresses)) + self.assertRaises(ipam_exc.IpAddressGenerationFailure, + ipam_subnet.bulk_allocate, + ipam_req.BulkAddressRequest(target_ip_count)) + def _test_deallocate_address(self, cidr, ip_version): ipam_subnet = self._create_and_allocate_ipam_subnet( cidr, ip_version=ip_version)[0]