Add IPManager class for handling IP addresses

This patch begins work on an 'IPManager' class that further decouples
IP/CIDR manipulation from the openstack-ansible configuration handling
logic.

With this patchset, a new implementation is provided and tested in
isolation, with integration into and replacement of existing code set
for follow up patchsets.

The aim is also to provide a working implementation based on a proposed
IP management API for use with plugins. The proposed API is provided as
the IPBasePlugin class, and generic expectations of the API are
documented there.

A few notable changes exist in the new IPManager class versus the
existing codebase:

    - The Queue.Queue class is not used, but rather a plain, randomized
    list. Reviewing the existing implementation, there does not appear
    to be a need to use the specialized queue class.

    - USED_IPS is moved into a set associated with a given IPManager
    object. The expectation is that dynamic_inventory.py will treat
    IPManager as a singleton, but this implementation allows for
    replacing that singleton in test cases, or using multiple instances
    in some other context.

While the lib/ip.py file is not intended to be executed, the python
shebang line was provided in order to comply with our tox linting
searches.

Change-Id: I06729ac2bc1688a39255f2c8ea0d14131b5c2560

lib/ip.py

@@ -1,7 +1,26 @@
+#!/usr/bin/env python
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# (c) 2014, Kevin Carter <kevin.carter@rackspace.com>
+# (c) 2016, Nolan Brubaker <nolan.brubaker@rackspace.com>
+
+import copy
+import logging
 import netaddr
 import Queue
 import random
-import logging
 
 logger = logging.getLogger('osa-inventory')
 
@@ -87,3 +106,217 @@ def set_used_ips(user_defined_config, inventory):
             if address:
                 logger.debug("IP %s set as used", address)
                 USED_IPS.add(address)
+
+
+class NoSuchQueue(Exception):
+    pass
+
+
+class EmptyQueue(Exception):
+    pass
+
+
+class IPBasePlugin(object):
+    def load(self, queue_name, cidr):
+        """Create a populate a queue with IP addresses
+
+        The network address and broadcast addresses should be excluded from
+        the IP addresses loaded into the queue.
+
+        Queue names should associate with their given CIDR. The queue values
+        should be a list of all available IP addresses based on CIDR range
+        and IP addresses already assigned.
+
+        """
+        raise NotImplementedError
+
+    def get(self, queue_name):
+        """Reserve an IP address from a given queue.
+
+        Should raise NoSuchQueue when the given queue name is not found,
+        and EmptyQueue if the queue is empty.
+
+        Some plugin implementations may be transaction, and require a call to
+        ``save`` after reserving an IP.
+        """
+        raise NotImplementedError
+
+    def release(self, ip):
+        """Release an IP back into queues as assignable.
+
+        Some plugin implementations may be transaction, and require a call to
+        ``save`` after releasing an IP.
+        """
+        raise NotImplementedError
+
+    def save(self):
+        """Write actions to data store
+
+        This method is optional to implement, and is presented as a hook for
+        use with transactional data stores.
+        """
+        raise NotImplementedError
+
+
+class IPManager(IPBasePlugin):
+    """Class to manage CIDRs and IPs from openstack-ansible inventory config
+
+    CIDRs are managed via queues, which will be named for convenience. All IP
+    addresses assigned are saved into the :method:`IPManager.used` set and
+    removed from their respective queue.
+
+    IP addresses that are no longer in use may be freed back into the queues
+    with the :method:`IPManager.release` method.
+
+    """
+    def __init__(self, queues=None, used_ips=None):
+        """Create a manager with various queues and a used IP blacklist
+
+        :param queues: ``dict`` A dictionary containing queue names for keys
+            and CIDR specifications for values.
+        :param used_ips: ``set`` A set of IP addresses which are marked as used
+            and unassignable. Any iterable will be coerced into a set to remove
+            duplicate entries.
+        """
+
+        if queues is None:
+            queues = {}
+
+        if used_ips is None:
+            used_ips = set()
+
+        # If we receive a set already, this is esentially a no-op,
+        # not a wrapper.
+        self._used_ips = set(used_ips)
+        self._queues = queues
+
+        # The networks will be netaddr.IPNetwork objects for a given CIDR,
+        # kept so that if an IP is released from use, it is returned to the
+        # associated queue.
+        self._networks = {}
+
+        # Populate any queues that were passed in already.
+        for name, cidr in queues.items():
+            self.load(name, cidr)
+
+    @property
+    def used(self):
+        """Set of IPs used within the environment
+
+        IP addresses within this set will be masked when requesting a new IP,
+        and thus not be returned to callers.
+
+        Set returned is a copy of the internal data structure.
+
+        :return: Set of IP addresses currently in use
+        :rtrype: set
+        """
+        return set(self._used_ips)
+
+    @used.deleter
+    def used(self):
+        """Empty the used IP set.
+
+        Any IP used will also be released back in to the associated
+        queue.
+        """
+        used_ips = set(self._used_ips)
+        for ip in used_ips:
+            self.release(ip)
+        self._used_ips = set()
+
+    @property
+    def queues(self):
+        """Dictionary of named queues, populated with IPs for a given CIDR.
+
+        Return values here are copies, to protect the internal structures
+        from unintentional changes.
+        """
+        return copy.deepcopy(self._queues)
+
+    def __getitem__(self, key):
+        """Short hand for accessing a named queue
+
+        The list returned is a copy of the internal queue.
+        """
+        return list(self._queues[key])
+
+    def load(self, queue_name, cidr):
+        """Populates a named queue with all IPs in a CIDR
+
+        Queues are implemented as a list, and will be populated by all IP
+        addresses within a CIDR, with the following exceptions:
+            * The network and broadcast IP addresses
+            * Any IP address already in the used_ips set
+
+        :param queue_name: ``str`` Name to apply to a given CIDR
+        :param cidr: ``str`` CIDR notation specifying range of IP addresses
+            which are available for assignment.
+        """
+        net = netaddr.IPNetwork(cidr)
+
+        initial_ips = [str(i) for i in list(net)]
+
+        # We will never want to assign these to machines.
+
+        if net.network:
+            self._used_ips.update([str(net.network)])
+        if net.broadcast:
+            self._used_ips.update([str(net.broadcast)])
+
+        all_ips = [ip for ip in initial_ips if ip not in self._used_ips]
+
+        # randomize so that we're not generating the expectation that
+        # groups are clustered by IP
+        random.shuffle(all_ips)
+
+        self._queues[queue_name] = all_ips
+        self._networks[queue_name] = net
+
+    def get(self, queue_name):
+        """Returns an usused IP address from a specified queue.
+
+        IPs returned will be marked as used and removed from the associated
+        queue.
+
+        :param queue_name: ``str`` Name of the queue from which to retrive an
+            IP.
+        :returns: IP address
+        :rtype: str
+        :raises: ip.NoSuchQueue, ip.EmptyQueue
+        """
+        if queue_name not in self._queues.keys():
+            raise NoSuchQueue("Queue {0} does not exist".format(queue_name))
+
+        try:
+            address = self._queues[queue_name].pop(0)
+        except IndexError:
+            raise EmptyQueue("Queue {0} is empty".format(queue_name))
+
+        self._used_ips.add(address)
+
+        return address
+
+    def release(self, ip):
+        """Free an IP from the used list and re-insert it to its queue.
+
+        Any IP freed will also be re-inserted into the associated queue, which
+        is calculated at deletion.
+
+        If an IP matches multiple CIDR ranges available, it will be inserted
+        to the first one matched.
+
+        :param ip: ``str`` IP address which to release back into the usable
+            pool.
+        """
+        self._used_ips.discard(ip)
+
+        # Use the IP class for membership comparison to the network
+        addr = netaddr.IPAddress(ip)
+
+        # TODO(nrb): Should this be ordered somehow to be more determinate?
+        # Alphabetical by queue name seems easiest, but not necessarily
+        # accurate or relevant.
+        for name, network in self._networks.items():
+            if addr in network:
+                self._queues[name].append(ip)

tests/test_ip.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+import os
+from os import path
+import sys
+import unittest
+
+LIB_DIR = path.join(os.getcwd(), 'lib')
+
+sys.path.append(LIB_DIR)
+
+import ip
+
+
+class TestIPManager(unittest.TestCase):
+    def test_basic_instantiation(self):
+        manager = ip.IPManager()
+
+        self.assertEqual({}, manager.queues)
+        self.assertEqual(set(), manager.used)
+
+    def test_verbose_instantiation(self):
+        manager = ip.IPManager(queues={'test': '192.168.0.0/24'},
+                               used_ips=set(['192.168.0.0', '192.168.0.255']))
+        self.assertEqual(2, len(manager.used))
+        self.assertEqual(254, len(manager.queues['test']))
+
+    def test__instantiation_with_used_list(self):
+        manager = ip.IPManager(used_ips=['192.168.0.0', '192.168.0.255'])
+
+        self.assertEqual(2, len(manager.used))
+
+    def test_verbose_instantiation_duplicated_ips(self):
+        manager = ip.IPManager(used_ips=['192.168.0.0', '192.168.0.0'])
+
+        self.assertEqual(1, len(manager.used))
+
+    def test_deleting_used(self):
+        manager = ip.IPManager(used_ips=set(['192.168.1.1']))
+
+        del manager.used
+
+        self.assertEqual(set(), manager.used)
+
+    def test_getitem(self):
+        manager = ip.IPManager(queues={'test': '192.168.0.0/24'})
+
+        self.assertEqual(manager.queues['test'], manager['test'])
+
+    def test_loading_queue(self):
+        manager = ip.IPManager()
+        manager.load('test', '192.168.0.0/24')
+        self.assertEqual(254, len(manager.queues['test']))
+
+    def test_loading_network_excludes(self):
+        manager = ip.IPManager()
+        manager.load('test', '192.168.0.0/24')
+        self.assertNotIn('192.168.0.0', manager.queues['test'])
+        self.assertNotIn('192.168.0.255', manager.queues['test'])
+
+    def test_loading_used_ips(self):
+        manager = ip.IPManager()
+        manager.load('test', '192.168.0.0/24')
+
+        self.assertEqual(2, len(manager.used))
+        self.assertIn('192.168.0.0', manager.used)
+        self.assertIn('192.168.0.255', manager.used)
+
+    def test_load_creates_networks(self):
+        manager = ip.IPManager()
+        manager.load('test', '192.168.0.0/24')
+
+        self.assertIn('test', manager._networks)
+
+    def test_loaded_randomly(self):
+        manager = ip.IPManager()
+        manager.load('test', '192.168.0.0/24')
+
+        self.assertNotEqual(['192.168.0.1', '192.168.0.2', '192.168.0.3'],
+                            manager.queues['test'][0:3])
+
+    def test_getting_ip(self):
+        manager = ip.IPManager(queues={'test': '192.168.0.0/24'})
+        my_ip = manager.get('test')
+
+        self.assertTrue(my_ip.startswith('192.168.0'))
+        self.assertIn(my_ip, manager.used)
+        self.assertNotIn(my_ip, manager.queues['test'])
+
+    def test_getting_ip_from_empty_queue(self):
+        manager = ip.IPManager(queues={'test': '192.168.0.0/31'})
+        # There will only be 1 usable IP address in this range.
+        manager.get('test')
+
+        with self.assertRaises(ip.EmptyQueue):
+            manager.get('test')
+
+    def test_get_ip_from_missing_queue(self):
+        manager = ip.IPManager()
+
+        with self.assertRaises(ip.NoSuchQueue):
+            manager.get('management')
+
+    def test_release_used_ip(self):
+        target_ip = '192.168.0.1'
+        manager = ip.IPManager(queues={'test': '192.168.0.0/31'},
+                               used_ips=[target_ip])
+
+        manager.release(target_ip)
+
+        # No broadcast address on this network, so only the network addr left
+        self.assertEqual(1, len(manager.used))
+        self.assertNotIn(target_ip, manager.used)
+        self.assertIn(target_ip, manager['test'])
+
+    def test_save_not_implemented(self):
+        manager = ip.IPManager()
+
+        with self.assertRaises(NotImplementedError):
+            manager.save()
+
+    def test_queue_dict_copied(self):
+        manager = ip.IPManager(queues={'test': '192.168.0.0/31'})
+        external = manager.queues
+        self.assertIsNot(manager.queues, external)
+        self.assertIsNot(manager.queues['test'], external['test'])
+
+    def test_queue_list_copied(self):
+        manager = ip.IPManager(queues={'test': '192.168.0.0/31'})
+        external = manager['test']
+        # test against the internal structure since .queues should
+        # itself be making copies
+        self.assertIsNot(manager._queues['test'], external)
+
+    def test_used_ips_copies(self):
+        manager = ip.IPManager(used_ips=['192.168.0.1'])
+        external = manager.used
+        self.assertIsNot(manager._used_ips, external)
+
+    def test_deleting_used_ips_releases_to_queues(self):
+        target_ip = '192.168.0.1'
+        manager = ip.IPManager(queues={'test': '192.168.0.0/31'},
+                               used_ips=[target_ip])
+
+        del manager.used
+
+        self.assertIn(target_ip, manager['test'])
+
+
+if __name__ == "__main__":
+    unittest.main()

tox.ini

@@ -152,6 +152,8 @@ commands =
     coverage erase
     coverage run {toxinidir}/tests/test_manage.py
     coverage report --show-missing --include={toxinidir}/lib/*
+    coverage run {toxinidir}/tests/test_ip.py
+    coverage report --show-missing --include={toxinidir}/lib/*
 
 
 [testenv:linters]