openstack-ansible/lib/ip.py
Nolan Brubaker 99e398125b Add IPManager class for handling IP addresses
This patch begins work on an 'IPManager' class that further decouples
IP/CIDR manipulation from the openstack-ansible configuration handling
logic.

With this patchset, a new implementation is provided and tested in
isolation, with integration into and replacement of existing code set
for follow up patchsets.

The aim is also to provide a working implementation based on a proposed
IP management API for use with plugins. The proposed API is provided as
the IPBasePlugin class, and generic expectations of the API are
documented there.

A few notable changes exist in the new IPManager class versus the
existing codebase:

    - The Queue.Queue class is not used, but rather a plain, randomized
    list. Reviewing the existing implementation, there does not appear
    to be a need to use the specialized queue class.

    - USED_IPS is moved into a set associated with a given IPManager
    object. The expectation is that dynamic_inventory.py will treat
    IPManager as a singleton, but this implementation allows for
    replacing that singleton in test cases, or using multiple instances
    in some other context.

While the lib/ip.py file is not intended to be executed, the python
shebang line was provided in order to comply with our tox linting
searches.

Change-Id: I06729ac2bc1688a39255f2c8ea0d14131b5c2560
2016-11-18 17:07:03 -05:00

323 lines
10 KiB
Python

#!/usr/bin/env python
# Copyright 2016, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# (c) 2014, Kevin Carter <kevin.carter@rackspace.com>
# (c) 2016, Nolan Brubaker <nolan.brubaker@rackspace.com>
import copy
import logging
import netaddr
import Queue
import random
logger = logging.getLogger('osa-inventory')
USED_IPS = set()
def get_ip_address(name, ip_q):
"""Return an IP address from our IP Address queue."""
try:
ip_addr = ip_q.get(timeout=1)
while ip_addr in USED_IPS:
ip_addr = ip_q.get(timeout=1)
else:
USED_IPS.add(ip_addr)
return str(ip_addr)
except AttributeError:
return None
except Queue.Empty:
raise SystemExit(
'Cannot retrieve requested amount of IP addresses. Increase the %s'
' range in your openstack_user_config.yml.' % name
)
def load_ip_q(cidr, ip_q):
"""Load the IP queue with all IP address from a given cidr.
:param cidr: ``str`` IP address with cidr notation
"""
_all_ips = [str(i) for i in list(netaddr.IPNetwork(cidr))]
base_exclude = [
str(netaddr.IPNetwork(cidr).network),
str(netaddr.IPNetwork(cidr).broadcast)
]
USED_IPS.update(base_exclude)
for ip in random.sample(_all_ips, len(_all_ips)):
if ip not in USED_IPS:
ip_q.put(ip)
def load_optional_q(config, cidr_name):
"""Load optional queue with ip addresses.
:param config: ``dict`` User defined information
:param cidr_name: ``str`` Name of the cidr name
"""
cidr = config.get(cidr_name)
ip_q = None
if cidr is not None:
ip_q = Queue.Queue()
load_ip_q(cidr=cidr, ip_q=ip_q)
return ip_q
def set_used_ips(user_defined_config, inventory):
"""Set all of the used ips into a global list.
:param user_defined_config: ``dict`` User defined configuration
:param inventory: ``dict`` Living inventory of containers and hosts
"""
used_ips = user_defined_config.get('used_ips')
if isinstance(used_ips, list):
for ip in used_ips:
split_ip = ip.split(',')
if len(split_ip) >= 2:
ip_range = list(
netaddr.iter_iprange(
split_ip[0],
split_ip[-1]
)
)
USED_IPS.update([str(i) for i in ip_range])
else:
logger.debug("IP %s set as used", split_ip[0])
USED_IPS.add(split_ip[0])
# Find all used IP addresses and ensure that they are not used again
for host_entry in inventory['_meta']['hostvars'].values():
networks = host_entry.get('container_networks', dict())
for network_entry in networks.values():
address = network_entry.get('address')
if address:
logger.debug("IP %s set as used", address)
USED_IPS.add(address)
class NoSuchQueue(Exception):
pass
class EmptyQueue(Exception):
pass
class IPBasePlugin(object):
def load(self, queue_name, cidr):
"""Create a populate a queue with IP addresses
The network address and broadcast addresses should be excluded from
the IP addresses loaded into the queue.
Queue names should associate with their given CIDR. The queue values
should be a list of all available IP addresses based on CIDR range
and IP addresses already assigned.
"""
raise NotImplementedError
def get(self, queue_name):
"""Reserve an IP address from a given queue.
Should raise NoSuchQueue when the given queue name is not found,
and EmptyQueue if the queue is empty.
Some plugin implementations may be transaction, and require a call to
``save`` after reserving an IP.
"""
raise NotImplementedError
def release(self, ip):
"""Release an IP back into queues as assignable.
Some plugin implementations may be transaction, and require a call to
``save`` after releasing an IP.
"""
raise NotImplementedError
def save(self):
"""Write actions to data store
This method is optional to implement, and is presented as a hook for
use with transactional data stores.
"""
raise NotImplementedError
class IPManager(IPBasePlugin):
"""Class to manage CIDRs and IPs from openstack-ansible inventory config
CIDRs are managed via queues, which will be named for convenience. All IP
addresses assigned are saved into the :method:`IPManager.used` set and
removed from their respective queue.
IP addresses that are no longer in use may be freed back into the queues
with the :method:`IPManager.release` method.
"""
def __init__(self, queues=None, used_ips=None):
"""Create a manager with various queues and a used IP blacklist
:param queues: ``dict`` A dictionary containing queue names for keys
and CIDR specifications for values.
:param used_ips: ``set`` A set of IP addresses which are marked as used
and unassignable. Any iterable will be coerced into a set to remove
duplicate entries.
"""
if queues is None:
queues = {}
if used_ips is None:
used_ips = set()
# If we receive a set already, this is esentially a no-op,
# not a wrapper.
self._used_ips = set(used_ips)
self._queues = queues
# The networks will be netaddr.IPNetwork objects for a given CIDR,
# kept so that if an IP is released from use, it is returned to the
# associated queue.
self._networks = {}
# Populate any queues that were passed in already.
for name, cidr in queues.items():
self.load(name, cidr)
@property
def used(self):
"""Set of IPs used within the environment
IP addresses within this set will be masked when requesting a new IP,
and thus not be returned to callers.
Set returned is a copy of the internal data structure.
:return: Set of IP addresses currently in use
:rtrype: set
"""
return set(self._used_ips)
@used.deleter
def used(self):
"""Empty the used IP set.
Any IP used will also be released back in to the associated
queue.
"""
used_ips = set(self._used_ips)
for ip in used_ips:
self.release(ip)
self._used_ips = set()
@property
def queues(self):
"""Dictionary of named queues, populated with IPs for a given CIDR.
Return values here are copies, to protect the internal structures
from unintentional changes.
"""
return copy.deepcopy(self._queues)
def __getitem__(self, key):
"""Short hand for accessing a named queue
The list returned is a copy of the internal queue.
"""
return list(self._queues[key])
def load(self, queue_name, cidr):
"""Populates a named queue with all IPs in a CIDR
Queues are implemented as a list, and will be populated by all IP
addresses within a CIDR, with the following exceptions:
* The network and broadcast IP addresses
* Any IP address already in the used_ips set
:param queue_name: ``str`` Name to apply to a given CIDR
:param cidr: ``str`` CIDR notation specifying range of IP addresses
which are available for assignment.
"""
net = netaddr.IPNetwork(cidr)
initial_ips = [str(i) for i in list(net)]
# We will never want to assign these to machines.
if net.network:
self._used_ips.update([str(net.network)])
if net.broadcast:
self._used_ips.update([str(net.broadcast)])
all_ips = [ip for ip in initial_ips if ip not in self._used_ips]
# randomize so that we're not generating the expectation that
# groups are clustered by IP
random.shuffle(all_ips)
self._queues[queue_name] = all_ips
self._networks[queue_name] = net
def get(self, queue_name):
"""Returns an usused IP address from a specified queue.
IPs returned will be marked as used and removed from the associated
queue.
:param queue_name: ``str`` Name of the queue from which to retrive an
IP.
:returns: IP address
:rtype: str
:raises: ip.NoSuchQueue, ip.EmptyQueue
"""
if queue_name not in self._queues.keys():
raise NoSuchQueue("Queue {0} does not exist".format(queue_name))
try:
address = self._queues[queue_name].pop(0)
except IndexError:
raise EmptyQueue("Queue {0} is empty".format(queue_name))
self._used_ips.add(address)
return address
def release(self, ip):
"""Free an IP from the used list and re-insert it to its queue.
Any IP freed will also be re-inserted into the associated queue, which
is calculated at deletion.
If an IP matches multiple CIDR ranges available, it will be inserted
to the first one matched.
:param ip: ``str`` IP address which to release back into the usable
pool.
"""
self._used_ips.discard(ip)
# Use the IP class for membership comparison to the network
addr = netaddr.IPAddress(ip)
# TODO(nrb): Should this be ordered somehow to be more determinate?
# Alphabetical by queue name seems easiest, but not necessarily
# accurate or relevant.
for name, network in self._networks.items():
if addr in network:
self._queues[name].append(ip)