openstack/openstack-ansible
Code Issues Proposed changes

Add IPManager class for handling IP addresses

Browse Source 
This patch begins work on an 'IPManager' class that further decouples
IP/CIDR manipulation from the openstack-ansible configuration handling
logic.

With this patchset, a new implementation is provided and tested in
isolation, with integration into and replacement of existing code set
for follow up patchsets.

The aim is also to provide a working implementation based on a proposed
IP management API for use with plugins. The proposed API is provided as
the IPBasePlugin class, and generic expectations of the API are
documented there.

A few notable changes exist in the new IPManager class versus the
existing codebase:

    - The Queue.Queue class is not used, but rather a plain, randomized
    list. Reviewing the existing implementation, there does not appear
    to be a need to use the specialized queue class.

    - USED_IPS is moved into a set associated with a given IPManager
    object. The expectation is that dynamic_inventory.py will treat
    IPManager as a singleton, but this implementation allows for
    replacing that singleton in test cases, or using multiple instances
    in some other context.

While the lib/ip.py file is not intended to be executed, the python
shebang line was provided in order to comply with our tox linting
searches.

Change-Id: I06729ac2bc1688a39255f2c8ea0d14131b5c2560
This commit is contained in:
Nolan Brubaker 2016-11-14 11:58:40 -05:00
parent 35691fef01
commit 99e398125b
3 changed files with 386 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

235
lib/ip.py
View File

@ -1,7 +1,26 @@
#!/usr/bin/env python
# Copyright 2016, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# (c) 2014, Kevin Carter <kevin.carter@rackspace.com>
# (c) 2016, Nolan Brubaker <nolan.brubaker@rackspace.com>
import copy
import logging
import netaddr
import Queue
import random
import logging
logger = logging.getLogger('osa-inventory')
@ -87,3 +106,217 @@ def set_used_ips(user_defined_config, inventory):
if address:
logger.debug("IP %s set as used", address)
USED_IPS.add(address)
class NoSuchQueue(Exception):
pass
class EmptyQueue(Exception):
pass
class IPBasePlugin(object):
def load(self, queue_name, cidr):
"""Create a populate a queue with IP addresses
The network address and broadcast addresses should be excluded from
the IP addresses loaded into the queue.
Queue names should associate with their given CIDR. The queue values
should be a list of all available IP addresses based on CIDR range
and IP addresses already assigned.
"""
raise NotImplementedError
def get(self, queue_name):
"""Reserve an IP address from a given queue.
Should raise NoSuchQueue when the given queue name is not found,
and EmptyQueue if the queue is empty.
Some plugin implementations may be transaction, and require a call to
``save`` after reserving an IP.
"""
raise NotImplementedError
def release(self, ip):
"""Release an IP back into queues as assignable.
Some plugin implementations may be transaction, and require a call to
``save`` after releasing an IP.
"""
raise NotImplementedError
def save(self):
"""Write actions to data store
This method is optional to implement, and is presented as a hook for
use with transactional data stores.
"""
raise NotImplementedError
class IPManager(IPBasePlugin):
"""Class to manage CIDRs and IPs from openstack-ansible inventory config
CIDRs are managed via queues, which will be named for convenience. All IP
addresses assigned are saved into the :method:`IPManager.used` set and
removed from their respective queue.
IP addresses that are no longer in use may be freed back into the queues
with the :method:`IPManager.release` method.
"""
def __init__(self, queues=None, used_ips=None):
"""Create a manager with various queues and a used IP blacklist
:param queues: ``dict`` A dictionary containing queue names for keys
and CIDR specifications for values.
:param used_ips: ``set`` A set of IP addresses which are marked as used
and unassignable. Any iterable will be coerced into a set to remove
duplicate entries.
"""
if queues is None:
queues = {}
if used_ips is None:
used_ips = set()
# If we receive a set already, this is esentially a no-op,
# not a wrapper.
self._used_ips = set(used_ips)
self._queues = queues
# The networks will be netaddr.IPNetwork objects for a given CIDR,
# kept so that if an IP is released from use, it is returned to the
# associated queue.
self._networks = {}
# Populate any queues that were passed in already.
for name, cidr in queues.items():
self.load(name, cidr)
@property
def used(self):
"""Set of IPs used within the environment
IP addresses within this set will be masked when requesting a new IP,
and thus not be returned to callers.
Set returned is a copy of the internal data structure.
:return: Set of IP addresses currently in use
:rtrype: set
"""
return set(self._used_ips)
@used.deleter
def used(self):
"""Empty the used IP set.
Any IP used will also be released back in to the associated
queue.
"""
used_ips = set(self._used_ips)
for ip in used_ips:
self.release(ip)
self._used_ips = set()
@property
def queues(self):
"""Dictionary of named queues, populated with IPs for a given CIDR.
Return values here are copies, to protect the internal structures
from unintentional changes.
"""
return copy.deepcopy(self._queues)
def __getitem__(self, key):
"""Short hand for accessing a named queue
The list returned is a copy of the internal queue.
"""
return list(self._queues[key])
def load(self, queue_name, cidr):
"""Populates a named queue with all IPs in a CIDR
Queues are implemented as a list, and will be populated by all IP
addresses within a CIDR, with the following exceptions:
* The network and broadcast IP addresses
* Any IP address already in the used_ips set
:param queue_name: ``str`` Name to apply to a given CIDR
:param cidr: ``str`` CIDR notation specifying range of IP addresses
which are available for assignment.
"""
net = netaddr.IPNetwork(cidr)
initial_ips = [str(i) for i in list(net)]
# We will never want to assign these to machines.
if net.network:
self._used_ips.update([str(net.network)])
if net.broadcast:
self._used_ips.update([str(net.broadcast)])
all_ips = [ip for ip in initial_ips if ip not in self._used_ips]
# randomize so that we're not generating the expectation that
# groups are clustered by IP
random.shuffle(all_ips)
self._queues[queue_name] = all_ips
self._networks[queue_name] = net
def get(self, queue_name):
"""Returns an usused IP address from a specified queue.
IPs returned will be marked as used and removed from the associated
queue.
:param queue_name: ``str`` Name of the queue from which to retrive an
IP.
:returns: IP address
:rtype: str
:raises: ip.NoSuchQueue, ip.EmptyQueue
"""
if queue_name not in self._queues.keys():
raise NoSuchQueue("Queue {0} does not exist".format(queue_name))
try:
address = self._queues[queue_name].pop(0)
except IndexError:
raise EmptyQueue("Queue {0} is empty".format(queue_name))
self._used_ips.add(address)
return address
def release(self, ip):
"""Free an IP from the used list and re-insert it to its queue.
Any IP freed will also be re-inserted into the associated queue, which
is calculated at deletion.
If an IP matches multiple CIDR ranges available, it will be inserted
to the first one matched.
:param ip: ``str`` IP address which to release back into the usable
pool.
"""
self._used_ips.discard(ip)
# Use the IP class for membership comparison to the network
addr = netaddr.IPAddress(ip)
# TODO(nrb): Should this be ordered somehow to be more determinate?
# Alphabetical by queue name seems easiest, but not necessarily
# accurate or relevant.
for name, network in self._networks.items():
if addr in network:
self._queues[name].append(ip)

150
tests/test_ip.py Normal file
View File

@ -0,0 +1,150 @@
#!/usr/bin/env python
import os
from os import path
import sys
import unittest
LIB_DIR = path.join(os.getcwd(), 'lib')
sys.path.append(LIB_DIR)
import ip
class TestIPManager(unittest.TestCase):
def test_basic_instantiation(self):
manager = ip.IPManager()
self.assertEqual({}, manager.queues)
self.assertEqual(set(), manager.used)
def test_verbose_instantiation(self):
manager = ip.IPManager(queues={'test': '192.168.0.0/24'},
used_ips=set(['192.168.0.0', '192.168.0.255']))
self.assertEqual(2, len(manager.used))
self.assertEqual(254, len(manager.queues['test']))
def test__instantiation_with_used_list(self):
manager = ip.IPManager(used_ips=['192.168.0.0', '192.168.0.255'])
self.assertEqual(2, len(manager.used))
def test_verbose_instantiation_duplicated_ips(self):
manager = ip.IPManager(used_ips=['192.168.0.0', '192.168.0.0'])
self.assertEqual(1, len(manager.used))
def test_deleting_used(self):
manager = ip.IPManager(used_ips=set(['192.168.1.1']))
del manager.used
self.assertEqual(set(), manager.used)
def test_getitem(self):
manager = ip.IPManager(queues={'test': '192.168.0.0/24'})
self.assertEqual(manager.queues['test'], manager['test'])
def test_loading_queue(self):
manager = ip.IPManager()
manager.load('test', '192.168.0.0/24')
self.assertEqual(254, len(manager.queues['test']))
def test_loading_network_excludes(self):
manager = ip.IPManager()
manager.load('test', '192.168.0.0/24')
self.assertNotIn('192.168.0.0', manager.queues['test'])
self.assertNotIn('192.168.0.255', manager.queues['test'])
def test_loading_used_ips(self):
manager = ip.IPManager()
manager.load('test', '192.168.0.0/24')
self.assertEqual(2, len(manager.used))
self.assertIn('192.168.0.0', manager.used)
self.assertIn('192.168.0.255', manager.used)
def test_load_creates_networks(self):
manager = ip.IPManager()
manager.load('test', '192.168.0.0/24')
self.assertIn('test', manager._networks)
def test_loaded_randomly(self):
manager = ip.IPManager()
manager.load('test', '192.168.0.0/24')
self.assertNotEqual(['192.168.0.1', '192.168.0.2', '192.168.0.3'],
manager.queues['test'][0:3])
def test_getting_ip(self):
manager = ip.IPManager(queues={'test': '192.168.0.0/24'})
my_ip = manager.get('test')
self.assertTrue(my_ip.startswith('192.168.0'))
self.assertIn(my_ip, manager.used)
self.assertNotIn(my_ip, manager.queues['test'])
def test_getting_ip_from_empty_queue(self):
manager = ip.IPManager(queues={'test': '192.168.0.0/31'})
# There will only be 1 usable IP address in this range.
manager.get('test')
with self.assertRaises(ip.EmptyQueue):
manager.get('test')
def test_get_ip_from_missing_queue(self):
manager = ip.IPManager()
with self.assertRaises(ip.NoSuchQueue):
manager.get('management')
def test_release_used_ip(self):
target_ip = '192.168.0.1'
manager = ip.IPManager(queues={'test': '192.168.0.0/31'},
used_ips=[target_ip])
manager.release(target_ip)
# No broadcast address on this network, so only the network addr left
self.assertEqual(1, len(manager.used))
self.assertNotIn(target_ip, manager.used)
self.assertIn(target_ip, manager['test'])
def test_save_not_implemented(self):
manager = ip.IPManager()
with self.assertRaises(NotImplementedError):
manager.save()
def test_queue_dict_copied(self):
manager = ip.IPManager(queues={'test': '192.168.0.0/31'})
external = manager.queues
self.assertIsNot(manager.queues, external)
self.assertIsNot(manager.queues['test'], external['test'])
def test_queue_list_copied(self):
manager = ip.IPManager(queues={'test': '192.168.0.0/31'})
external = manager['test']
# test against the internal structure since .queues should
# itself be making copies
self.assertIsNot(manager._queues['test'], external)
def test_used_ips_copies(self):
manager = ip.IPManager(used_ips=['192.168.0.1'])
external = manager.used
self.assertIsNot(manager._used_ips, external)
def test_deleting_used_ips_releases_to_queues(self):
target_ip = '192.168.0.1'
manager = ip.IPManager(queues={'test': '192.168.0.0/31'},
used_ips=[target_ip])
del manager.used
self.assertIn(target_ip, manager['test'])
if __name__ == "__main__":
unittest.main()

2
tox.ini
View File

@ -152,6 +152,8 @@ commands =
coverage erase
coverage run {toxinidir}/tests/test_manage.py
coverage report --show-missing --include={toxinidir}/lib/*
coverage run {toxinidir}/tests/test_ip.py
coverage report --show-missing --include={toxinidir}/lib/*
[testenv:linters]