diff --git a/lib/ip.py b/lib/ip.py index 3ab0babfad..cac99689fa 100644 --- a/lib/ip.py +++ b/lib/ip.py @@ -1,7 +1,26 @@ +#!/usr/bin/env python +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2014, Kevin Carter +# (c) 2016, Nolan Brubaker + +import copy +import logging import netaddr import Queue import random -import logging logger = logging.getLogger('osa-inventory') @@ -87,3 +106,217 @@ def set_used_ips(user_defined_config, inventory): if address: logger.debug("IP %s set as used", address) USED_IPS.add(address) + + +class NoSuchQueue(Exception): + pass + + +class EmptyQueue(Exception): + pass + + +class IPBasePlugin(object): + def load(self, queue_name, cidr): + """Create a populate a queue with IP addresses + + The network address and broadcast addresses should be excluded from + the IP addresses loaded into the queue. + + Queue names should associate with their given CIDR. The queue values + should be a list of all available IP addresses based on CIDR range + and IP addresses already assigned. + + """ + raise NotImplementedError + + def get(self, queue_name): + """Reserve an IP address from a given queue. + + Should raise NoSuchQueue when the given queue name is not found, + and EmptyQueue if the queue is empty. + + Some plugin implementations may be transaction, and require a call to + ``save`` after reserving an IP. + """ + raise NotImplementedError + + def release(self, ip): + """Release an IP back into queues as assignable. + + Some plugin implementations may be transaction, and require a call to + ``save`` after releasing an IP. + """ + raise NotImplementedError + + def save(self): + """Write actions to data store + + This method is optional to implement, and is presented as a hook for + use with transactional data stores. + """ + raise NotImplementedError + + +class IPManager(IPBasePlugin): + """Class to manage CIDRs and IPs from openstack-ansible inventory config + + CIDRs are managed via queues, which will be named for convenience. All IP + addresses assigned are saved into the :method:`IPManager.used` set and + removed from their respective queue. + + IP addresses that are no longer in use may be freed back into the queues + with the :method:`IPManager.release` method. + + """ + def __init__(self, queues=None, used_ips=None): + """Create a manager with various queues and a used IP blacklist + + :param queues: ``dict`` A dictionary containing queue names for keys + and CIDR specifications for values. + :param used_ips: ``set`` A set of IP addresses which are marked as used + and unassignable. Any iterable will be coerced into a set to remove + duplicate entries. + """ + + if queues is None: + queues = {} + + if used_ips is None: + used_ips = set() + + # If we receive a set already, this is esentially a no-op, + # not a wrapper. + self._used_ips = set(used_ips) + self._queues = queues + + # The networks will be netaddr.IPNetwork objects for a given CIDR, + # kept so that if an IP is released from use, it is returned to the + # associated queue. + self._networks = {} + + # Populate any queues that were passed in already. + for name, cidr in queues.items(): + self.load(name, cidr) + + @property + def used(self): + """Set of IPs used within the environment + + IP addresses within this set will be masked when requesting a new IP, + and thus not be returned to callers. + + Set returned is a copy of the internal data structure. + + :return: Set of IP addresses currently in use + :rtrype: set + """ + return set(self._used_ips) + + @used.deleter + def used(self): + """Empty the used IP set. + + Any IP used will also be released back in to the associated + queue. + """ + used_ips = set(self._used_ips) + for ip in used_ips: + self.release(ip) + self._used_ips = set() + + @property + def queues(self): + """Dictionary of named queues, populated with IPs for a given CIDR. + + Return values here are copies, to protect the internal structures + from unintentional changes. + """ + return copy.deepcopy(self._queues) + + def __getitem__(self, key): + """Short hand for accessing a named queue + + The list returned is a copy of the internal queue. + """ + return list(self._queues[key]) + + def load(self, queue_name, cidr): + """Populates a named queue with all IPs in a CIDR + + Queues are implemented as a list, and will be populated by all IP + addresses within a CIDR, with the following exceptions: + * The network and broadcast IP addresses + * Any IP address already in the used_ips set + + :param queue_name: ``str`` Name to apply to a given CIDR + :param cidr: ``str`` CIDR notation specifying range of IP addresses + which are available for assignment. + """ + net = netaddr.IPNetwork(cidr) + + initial_ips = [str(i) for i in list(net)] + + # We will never want to assign these to machines. + + if net.network: + self._used_ips.update([str(net.network)]) + if net.broadcast: + self._used_ips.update([str(net.broadcast)]) + + all_ips = [ip for ip in initial_ips if ip not in self._used_ips] + + # randomize so that we're not generating the expectation that + # groups are clustered by IP + random.shuffle(all_ips) + + self._queues[queue_name] = all_ips + self._networks[queue_name] = net + + def get(self, queue_name): + """Returns an usused IP address from a specified queue. + + IPs returned will be marked as used and removed from the associated + queue. + + :param queue_name: ``str`` Name of the queue from which to retrive an + IP. + :returns: IP address + :rtype: str + :raises: ip.NoSuchQueue, ip.EmptyQueue + """ + if queue_name not in self._queues.keys(): + raise NoSuchQueue("Queue {0} does not exist".format(queue_name)) + + try: + address = self._queues[queue_name].pop(0) + except IndexError: + raise EmptyQueue("Queue {0} is empty".format(queue_name)) + + self._used_ips.add(address) + + return address + + def release(self, ip): + """Free an IP from the used list and re-insert it to its queue. + + Any IP freed will also be re-inserted into the associated queue, which + is calculated at deletion. + + If an IP matches multiple CIDR ranges available, it will be inserted + to the first one matched. + + :param ip: ``str`` IP address which to release back into the usable + pool. + """ + self._used_ips.discard(ip) + + # Use the IP class for membership comparison to the network + addr = netaddr.IPAddress(ip) + + # TODO(nrb): Should this be ordered somehow to be more determinate? + # Alphabetical by queue name seems easiest, but not necessarily + # accurate or relevant. + for name, network in self._networks.items(): + if addr in network: + self._queues[name].append(ip) diff --git a/tests/test_ip.py b/tests/test_ip.py new file mode 100644 index 0000000000..ac54225ca7 --- /dev/null +++ b/tests/test_ip.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python +import os +from os import path +import sys +import unittest + +LIB_DIR = path.join(os.getcwd(), 'lib') + +sys.path.append(LIB_DIR) + +import ip + + +class TestIPManager(unittest.TestCase): + def test_basic_instantiation(self): + manager = ip.IPManager() + + self.assertEqual({}, manager.queues) + self.assertEqual(set(), manager.used) + + def test_verbose_instantiation(self): + manager = ip.IPManager(queues={'test': '192.168.0.0/24'}, + used_ips=set(['192.168.0.0', '192.168.0.255'])) + self.assertEqual(2, len(manager.used)) + self.assertEqual(254, len(manager.queues['test'])) + + def test__instantiation_with_used_list(self): + manager = ip.IPManager(used_ips=['192.168.0.0', '192.168.0.255']) + + self.assertEqual(2, len(manager.used)) + + def test_verbose_instantiation_duplicated_ips(self): + manager = ip.IPManager(used_ips=['192.168.0.0', '192.168.0.0']) + + self.assertEqual(1, len(manager.used)) + + def test_deleting_used(self): + manager = ip.IPManager(used_ips=set(['192.168.1.1'])) + + del manager.used + + self.assertEqual(set(), manager.used) + + def test_getitem(self): + manager = ip.IPManager(queues={'test': '192.168.0.0/24'}) + + self.assertEqual(manager.queues['test'], manager['test']) + + def test_loading_queue(self): + manager = ip.IPManager() + manager.load('test', '192.168.0.0/24') + self.assertEqual(254, len(manager.queues['test'])) + + def test_loading_network_excludes(self): + manager = ip.IPManager() + manager.load('test', '192.168.0.0/24') + self.assertNotIn('192.168.0.0', manager.queues['test']) + self.assertNotIn('192.168.0.255', manager.queues['test']) + + def test_loading_used_ips(self): + manager = ip.IPManager() + manager.load('test', '192.168.0.0/24') + + self.assertEqual(2, len(manager.used)) + self.assertIn('192.168.0.0', manager.used) + self.assertIn('192.168.0.255', manager.used) + + def test_load_creates_networks(self): + manager = ip.IPManager() + manager.load('test', '192.168.0.0/24') + + self.assertIn('test', manager._networks) + + def test_loaded_randomly(self): + manager = ip.IPManager() + manager.load('test', '192.168.0.0/24') + + self.assertNotEqual(['192.168.0.1', '192.168.0.2', '192.168.0.3'], + manager.queues['test'][0:3]) + + def test_getting_ip(self): + manager = ip.IPManager(queues={'test': '192.168.0.0/24'}) + my_ip = manager.get('test') + + self.assertTrue(my_ip.startswith('192.168.0')) + self.assertIn(my_ip, manager.used) + self.assertNotIn(my_ip, manager.queues['test']) + + def test_getting_ip_from_empty_queue(self): + manager = ip.IPManager(queues={'test': '192.168.0.0/31'}) + # There will only be 1 usable IP address in this range. + manager.get('test') + + with self.assertRaises(ip.EmptyQueue): + manager.get('test') + + def test_get_ip_from_missing_queue(self): + manager = ip.IPManager() + + with self.assertRaises(ip.NoSuchQueue): + manager.get('management') + + def test_release_used_ip(self): + target_ip = '192.168.0.1' + manager = ip.IPManager(queues={'test': '192.168.0.0/31'}, + used_ips=[target_ip]) + + manager.release(target_ip) + + # No broadcast address on this network, so only the network addr left + self.assertEqual(1, len(manager.used)) + self.assertNotIn(target_ip, manager.used) + self.assertIn(target_ip, manager['test']) + + def test_save_not_implemented(self): + manager = ip.IPManager() + + with self.assertRaises(NotImplementedError): + manager.save() + + def test_queue_dict_copied(self): + manager = ip.IPManager(queues={'test': '192.168.0.0/31'}) + external = manager.queues + self.assertIsNot(manager.queues, external) + self.assertIsNot(manager.queues['test'], external['test']) + + def test_queue_list_copied(self): + manager = ip.IPManager(queues={'test': '192.168.0.0/31'}) + external = manager['test'] + # test against the internal structure since .queues should + # itself be making copies + self.assertIsNot(manager._queues['test'], external) + + def test_used_ips_copies(self): + manager = ip.IPManager(used_ips=['192.168.0.1']) + external = manager.used + self.assertIsNot(manager._used_ips, external) + + def test_deleting_used_ips_releases_to_queues(self): + target_ip = '192.168.0.1' + manager = ip.IPManager(queues={'test': '192.168.0.0/31'}, + used_ips=[target_ip]) + + del manager.used + + self.assertIn(target_ip, manager['test']) + + +if __name__ == "__main__": + unittest.main() diff --git a/tox.ini b/tox.ini index 27d61544b9..e8f1b76782 100644 --- a/tox.ini +++ b/tox.ini @@ -152,6 +152,8 @@ commands = coverage erase coverage run {toxinidir}/tests/test_manage.py coverage report --show-missing --include={toxinidir}/lib/* + coverage run {toxinidir}/tests/test_ip.py + coverage report --show-missing --include={toxinidir}/lib/* [testenv:linters]