openstack/fuel-devops
Code Issues Proposed changes

Refactoring. Some intermediate state.

Browse Source
This commit is contained in:
vic
2013-01-04 09:14:36 +04:00
parent fbed928252
commit bd537bde72
38 changed files with 619 additions and 750 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
devops/__init__.py
View File

@@ -1 +0,0 @@
from .main import *

326
devops/controller.py
View File

@@ -1,326 +0,0 @@
import os
import sys
import stat
import tempfile
import shutil
import urllib
import ipaddr
import glob
import random
import string
import re
import time
from model import Node, Network
from network import IpNetworksPool
from error import DevopsError
import my_yaml
import logging
logger = logging.getLogger('devops.controller')
def randstr(length=8):
return ''.join(random.choice(string.ascii_letters) for i in xrange(length))
class Controller:
def __init__(self, driver):
self.driver = driver
self.networks_pool = IpNetworksPool()
self._reserve_networks()
self.home_dir = os.environ.get('DEVOPS_HOME')
if self.home_dir is None:
home_dir = os.environ.get('APPDATA') or os.environ['HOME']
self.home_dir = os.path.join(home_dir,'.devops')
try:
os.makedirs(os.path.join(self.home_dir, 'environments'), 0755)
except OSError:
sys.exc_clear()
def build_environment(self, environment):
logger.info("Building environment %s" % environment.name)
environment.id = environment.name
logger.debug(
"Creating environment working directory for %s environment" % environment.name)
environment.work_dir = os.path.join(self.home_dir, 'environments',
environment.id)
if os.path.isdir(environment.work_dir):
if os.listdir(environment.work_dir):
raise DevopsError(
"Working directory already exists and not emtpy: %s" % environment.work_dir)
if not os.path.isdir(environment.work_dir):
os.makedirs(environment.work_dir)
logger.debug(
"Environment working directory has been created: %s" % environment.work_dir)
environment.driver = self.driver
for node in environment.nodes:
node.environment = environment
for network in environment.networks:
network.environment = environment
for node in environment.nodes:
for interface in node.interfaces:
interface.node = node
interface.network.interfaces.append(interface)
logger.info("Calculated interfaces '%s' '%s'" % (interface.node, interface.network.name))
for disk in node.disks:
if disk.base_image and disk.base_image.find('://') != -1:
disk.base_image = self._cache_file(disk.base_image)
if node.cdrom:
if node.cdrom.isopath.find('://') != -1:
node.cdrom.isopath = self._cache_file(node.cdrom.isopath)
for network in environment.networks:
logger.info("Building network %s" % network.name)
network.ip_addresses = self.networks_pool.get()
if network.pxe:
network.dhcp_server = True
tftp_path = os.path.join(environment.work_dir, "tftp")
if not os.path.exists(tftp_path):
os.mkdir(tftp_path,
stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
network.tftp_root_dir = tftp_path
if network.dhcp_server or network.reserve_static:
allocated_addresses = []
for interface in network.interfaces:
for address in interface.ip_addresses:
if address in network.ip_addresses:
allocated_addresses.append(address)
logger.info("Allocate addresses are calculated for '%s'" % network.name)
dhcp_allowed_addresses = list(network.ip_addresses)[2:-2]
for interface in network.interfaces:
logger.info("Enumerated interfaces '%s' '%s'" % (interface.node, interface.network.name))
logger.info(list(interface.ip_addresses))
if not len(list(interface.ip_addresses)):
address = self.get_first_free_address(
dhcp_allowed_addresses,
allocated_addresses)
if address is None:
raise DevopsError, "Failed to allocate IP address for node '%s' in network '%s': no more addresses left" % (interface.node.name, network.name)
interface.ip_addresses.append(address)
logger.info("Allocate IP address '%s' for node '%s' in network '%s'" % (address, interface.node.name, network.name))
allocated_addresses.append(address)
network.dhcp_dynamic_address_start = dhcp_allowed_addresses[0]
network.dhcp_dynamic_address_end = dhcp_allowed_addresses[-1]
for network in environment.networks:
logger.info("Building network %s" % network.name)
self.driver.create_network(network)
network.driver = self.driver
for node in environment.nodes:
logger.info("Building node %s" % node.name)
self._build_node(node)
node.driver = self.driver
for network in environment.networks:
self.driver.create_network(network)
network.start()
environment.built = True
logger.info("Finished building environment %s" % environment.name)
def destroy_environment(self, environment):
logger.info("Destroying environment %s" % environment.name)
for node in environment.nodes:
logger.info("Destroying node %s" % node.name)
node.stop()
for snapshot in node.snapshots:
self.driver.delete_snapshot(node, snapshot)
for disk in node.disks:
self.driver.delete_disk(disk)
self.driver.delete_node(node)
del node.driver
for network in environment.networks:
logger.info("Destroying network %s" % network.name)
network.stop()
self.driver.delete_network(network)
del network.driver
# FIXME
try:
self.networks_pool.put(network.ip_addresses)
except:
pass
del environment.driver
logger.info("Removing environment %s files" % environment.name)
shutil.rmtree(environment.work_dir)
logger.info("Finished destroying environment %s" % environment.name)
def load_environment(self, environment_id):
env_work_dir = os.path.join(
self.home_dir, 'environments',
environment_id)
env_config_file = os.path.join(env_work_dir, 'config')
if not os.path.exists(env_config_file):
raise DevopsError, "Environment '%s' couldn't be found" % environment_id
with file(env_config_file) as f:
data = f.read()
environment = my_yaml.load(data)
return environment
def save_environment(self, environment):
data = my_yaml.dump(environment)
if not environment.built:
raise DevopsError, "Environment has not been built yet."
with file(os.path.join(environment.work_dir, 'config'), 'w') as f:
f.write(data)
@property
def saved_environments(self):
saved_environments = []
for path in glob.glob(os.path.join(self.home_dir, 'environments', '*')):
if os.path.exists(os.path.join(path, 'config')):
saved_environments.append(os.path.basename(path))
return saved_environments
def _reserve_networks(self):
logger.debug("Scanning for ip networks that are already taken")
allocated_networks = self.driver.get_allocated_networks()
if allocated_networks:
for network in allocated_networks:
logger.debug("Reserving ip network %s" % network)
self.networks_pool.reserve(ipaddr.IPNetwork(network))
# todo use settings
with os.popen("ip route") as f:
for line in f:
words = line.split()
if len(words) == 0:
continue
if words[0] == 'default':
continue
address = ipaddr.IPv4Network(words[0])
logger.debug("Reserving ip network %s" % address)
self.networks_pool.reserve(address)
logger.debug("Finished scanning for taken ip networks")
def _build_node(self, node):
for disk in filter(lambda d: d.path is None, node.disks):
logger.debug("Creating disk file for node '%s'" % node.name)
disk.path = self.driver.create_disk(disk)
logger.debug("Creating node '%s'" % node.name)
self.driver.create_node(node)
def _cache_file(self, url):
cache_dir = os.path.join(self.home_dir, 'cache')
if not os.path.exists(cache_dir):
os.makedirs(cache_dir, 0755)
cache_log_path = os.path.join(cache_dir, 'entries')
if os.path.exists(cache_log_path):
with file(cache_log_path) as f:
cache_entries = my_yaml.load(f.read())
else:
cache_entries = dict()
ext_cache_log_path = os.path.join(cache_dir, 'extended_entries')
if os.path.exists(ext_cache_log_path):
with file(ext_cache_log_path) as f:
extended_cache_entries = my_yaml.load(f.read())
else:
extended_cache_entries = dict()
for key, value in cache_entries.items():
if not extended_cache_entries.has_key(key):
extended_cache_entries[key] = {'cached-path': value}
RFC1123_DATETIME_FORMAT = '%a, %d %b %Y %H:%M:%S %Z'
url_attrs = {}
cached_path = ''
local_mtime = 0
if extended_cache_entries.has_key(url):
url_attrs = extended_cache_entries[url]
cached_path = url_attrs['cached-path']
if url_attrs.has_key('last-modified'):
local_mtime = time.mktime(
time.strptime(url_attrs['last-modified'],
RFC1123_DATETIME_FORMAT))
else:
logger.debug("Cache miss for '%s', downloading" % url)
remote = urllib.urlopen(url)
msg = remote.info()
if msg.has_key('last-modified'):
url_mtime = time.mktime(
time.strptime(msg['last-modified'], RFC1123_DATETIME_FORMAT))
else:
url_mtime = 0
if local_mtime >= url_mtime:
logger.debug("Cache is up to date for '%s': '%s'" % (
url, cached_path))
return cached_path
elif cached_path != '':
logger.debug("Cache is old for '%s', downloading" % url)
if not os.access(cached_path, os.W_OK):
try:
os.unlink(cached_path)
except Exception:
pass
fd, cached_path = tempfile.mkstemp(prefix=cache_dir + '/')
os.close(fd)
with file(cached_path, 'w') as f:
shutil.copyfileobj(remote, f)
os.chmod(
cached_path,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
if msg.has_key('last-modified'):
url_attrs['last-modified'] = msg['last-modified']
url_attrs['cached-path'] = cached_path
extended_cache_entries[url] = url_attrs
with file(ext_cache_log_path, 'w') as f:
f.write(my_yaml.dump(extended_cache_entries))
cache_entries[url] = cached_path
with file(cache_log_path, 'w') as f:
f.write(my_yaml.dump(cache_entries))
logger.debug("Cached '%s' to '%s'" % (url, cached_path))
return cached_path
def get_first_free_address(self, allowed_addresses, allocated_addresses):
s = set(allocated_addresses)
for x in allowed_addresses:
if x not in s:
return x
return None

15
devops/libvirt_cleanup
View File

@@ -1,15 +0,0 @@
#!/bin/bash
function virsh { command virsh "$@" |grev -v "jenkins-"|awk 'NR>2 && $0!=""'; }
function foreach { xargs --no-run-if-empty -n 1 "$@"; }
function column { awk "{print \$$1}"; }
virsh list --all | column 2 | foreach virsh destroy
virsh list --all | column 2 | while read nodeid; do
virsh snapshot-list $nodeid | column 1 | foreach virsh snapshot-delete $nodeid
virsh undefine $nodeid
done
virsh net-list --all | column 1 | foreach virsh net-destroy
virsh net-list --all | column 1 | foreach virsh net-undefine

45
devops/main.py
View File

@@ -1,45 +0,0 @@
import logging
from error import DevopsError
from controller import Controller
from driver.libvirt import Libvirt
import yaml_config_loader
__all__ = ['logger', 'getController', 'build', 'destroy', 'load', 'save']
logger = logging.getLogger('devops')
class ControllerSingleton(Controller):
_instance = None
def __new__(cls, *args, **kwargs):
if not cls._instance:
cls._instance = super(ControllerSingleton, cls).__new__(
cls, *args, **kwargs)
return cls._instance
def getController():
return ControllerSingleton(Libvirt())
def build(environment):
getController().build_environment(environment)
def destroy(environment):
getController().destroy_environment(environment)
def load(source):
source = str(source).strip()
if source.find("\n") == -1:
if not source in getController().saved_environments:
raise DevopsError, "Environment '%s' does not exist" % source
return getController().load_environment(source)
return yaml_config_loader.load(source)
def save(environment):
getController().save_environment(environment)

46
devops/my_yaml.py
View File

@@ -1,46 +0,0 @@
import yaml
import ipaddr
def ipv4_address_constructor(loader, node):
return loader.make_python_instance(
'ipaddr.IPv4Address', node,
args=[node.value])
def ipv4_network_constructor(loader, node):
return loader.make_python_instance(
'ipaddr.IPv4Network', node,
args=[node.value])
def python_object_string_representer(dumper, data):
full_class_name = data.__class__.__name__
if data.__module__:
full_class_name = data.__module__ + '.' + full_class_name
return yaml.ScalarNode(
"tag:yaml.org,2002:python/object:%s" % full_class_name, str(data),
style=None)
class Loader(yaml.Loader):
pass
class Dumper(yaml.Dumper):
pass
Loader.add_constructor('tag:yaml.org,2002:python/object:ipaddr.IPv4Address',
ipv4_address_constructor)
Loader.add_constructor('tag:yaml.org,2002:python/object:ipaddr.IPv4Network',
ipv4_network_constructor)
Dumper.add_representer(ipaddr.IPv4Address, python_object_string_representer)
Dumper.add_representer(ipaddr.IPv4Network, python_object_string_representer)
def load(stream):
return yaml.load(stream, Loader=Loader)
def dump(data):
return yaml.dump(data, Dumper=Dumper)

52
devops/network.py
View File

@@ -1,52 +0,0 @@
import ipaddr
from itertools import chain
IPv4Address = ipaddr.IPv4Address
IPv4Network = ipaddr.IPv4Network
class NetworkPoolException(Exception):
pass
class IpNetworksPool:
def __init__(self, net_addresses=None, prefix=24):
if not net_addresses:
net_addresses = ['10.0.0.0/16']
networks = []
for address in net_addresses:
if not isinstance(address, IPv4Network):
address = IPv4Network(str(address))
networks.append(address)
self._available_networks = set(chain(
*[net_address.iter_subnets(new_prefix=prefix) for net_address in
networks]))
self._allocated_networks = set()
def reserve(self, network):
for overlaping_network in filter(
lambda n: n.overlaps(network),
self._available_networks
):
self._available_networks.remove(overlaping_network)
def get(self):
"get() - allocates and returns network address"
x = self._available_networks.pop()
self._allocated_networks.add(x)
return x
def put(self, network):
"put(net_address) - return network address to pool"
x = network
if x not in self._allocated_networks:
raise NetworkPoolException, "Network address '%s' wasn't previously allocated" % str(
network)
self._allocated_networks.remove(x)
self._available_networks.add(x)
@property
def is_empty(self):
return len(self._available_networks) == 0

50
devops/xml.py
View File

@@ -1,50 +0,0 @@
from lxml import etree
class Element:
def __init__(self, element):
self.element = element
self.tag = self.element.tag
self.text = self.element.text
def __getitem__(self, name, default=None):
return self.element.get(name, default)
def __setitem__(self, name, value):
return self.element.set(name, value)
def find_all(self, xpath):
"find_all(xpath) - returns list of elements matching given XPath"
results = []
for e in self.element.xpath(xpath):
if hasattr(e, 'xpath'):
# wrap element nodes with our class
e = Element(e)
else:
e = str(e)
results.append(e)
return results
def find(self, xpath):
"find(xpath) - returns first element matching given XPath or None"
elements = self.find_all(xpath)
if len(elements) == 0:
return None
return elements[0]
def parse_file(path):
"""parse_file(path) - parse file contents as XML and return XML document object."""
with file(path) as f:
return parse_stream(f)
def parse_stream(stream):
"""parse_stream(stream) - parse stream as XML and return XML document object."""
return parse_string(stream.read())
def parse_string(s):
"parse_string(s) - parse string as XML and return XML document object."
return Element(etree.fromstring(s))

170
devops/yaml_config_loader.py
View File

@@ -1,170 +0,0 @@
import re
import my_yaml
from model import Environment, Network, Node, Disk, Interface, Cdrom
class ConfigError(Exception):
pass
def load(stream):
data = my_yaml.load(stream)
if not data.has_key('nodes'):
raise ConfigError, "No nodes defined"
name = 'default'
if data.has_key('name'):
name = data['name']
environment = Environment(name)
for network_data in (data.get('networks') or []):
parse_network(environment, network_data)
for node_data in data['nodes']:
parse_node(environment, node_data)
return environment
def dump(stream):
raise "Not implemented yet"
def parse_network(environment, data):
if data.has_key('name'):
name = data['name']
elif data.has_key('network'):
name = data['network']
else:
raise ConfigError, "Unnamed network"
network = Network(name)
if data.has_key('dhcp_server'):
network.dhcp_server = data['dhcp_server']
for existing_network in environment.networks:
if existing_network.name == network.name:
raise ConfigError, "Network with given name already exists: %s" % network.name
environment.networks.append(network)
return network
def parse_node(environment, data):
if data.has_key('name'):
name = data['name']
elif data.has_key('node'):
name = data['node']
else:
raise ConfigError, "Unnamed node"
node = Node(name)
if data.has_key('cpu'):
node.cpu = data['cpu']
if data.has_key('memory'):
node.memory = data['memory']
if data.has_key('vnc'):
node.vnc = data['vnc']
if data.has_key('cdrom'):
isopath = data['cdrom']
if not isinstance(isopath, (str,)):
raise ConfigError, "It must be string containing path to iso image file"
node.cdrom = Cdrom(isopath)
if data.has_key('disk'):
disks_data = data['disk']
if type(disks_data) != list:
disks_data = (disks_data,)
for disk_data in disks_data:
if type(disk_data) == str:
try:
size = parse_size(disk_data)
node.disks.append(Disk(size=size))
except ValueError:
path = disk_data
node.disks.append(Disk(path=path))
elif isinstance(disk_data, dict):
node.disks.append(Disk(**disk_data))
else:
raise ConfigError, "Unknown disk config: %s" % str(disk_data)
if data.has_key('networks'):
networks_data = data['networks']
if type(networks_data) != list:
networks_data = (networks_data,)
for network_data in networks_data:
if type(network_data) == str:
network = None
for n in environment.networks:
if n.name == network_data:
network = n
break
# Inline networks
# if network is None:
# network = parse_network(environment, {'name': network_data})
# self.networks.append(network)
# TODO: add support for specifying additional network interface params (e.g. mac address)
if network is None:
raise ConfigError, "Unknown network %s" % network_data
node.interfaces.append(Interface(network))
if data.has_key('boot'):
boot_data = data['boot']
if type(boot_data) != list:
boot_data = list(boot_data)
for boot in boot_data:
if not boot in ('disk', 'network', 'cdrom'):
raise ConfigError, "Unknown boot option: %s" % boot
node.boot.append(boot)
else:
if len(node.disks) > 0:
node.boot.append('disk')
if node.cdrom:
node.boot.append('cdrom')
if len(node.interfaces) > 0:
node.boot.append('network')
for existing_node in environment.nodes:
if existing_node.name == node.name:
raise ConfigError, "Node with given name already exists: %s" % node.name
environment.nodes.append(node)
SIZE_RE = re.compile('^(\d+)\s*(|kb|k|mb|m|gb|g)$')
def parse_size(s):
m = SIZE_RE.match(s.lower())
if not m:
raise ValueError, "Invalid size format: %s" % s
value = int(m.group(1))
units = m.group(2)
if units in ['k', 'kb']:
multiplier = 1024
elif units in ['m', 'mb']:
multiplier = 1024 ** 2
elif units in ['g', 'gb']:
multiplier = 1024 ** 3
elif units in ['t', 'tb']:
multiplier = 1024 ** 4
elif units == '':
multiplier = 1
else:
raise ValueError, "Invalid size format: %s" % units
return value * multiplier

9
samples/install_admin.py
View File

@@ -7,11 +7,10 @@ import time
import shutil
from optparse import OptionParser
from devops.model import Environment, Network, Node, Disk, Cdrom, Interface
from devops.controller import Controller
from devops.network import IpNetworksPool
from devops.driver.libvirt import Libvirt
from devops.helpers import tcp_ping, wait
from src.devops.model import Environment, Network, Node, Disk, Cdrom, Interface
from src.devops.controller import Controller
from src.devops.driver.libvirt import Libvirt
from src.devops.helpers import tcp_ping, wait
ADMIN_ISO_NAME = 'nailgun-ubuntu-12.04-amd64.last.iso'
ADMIN_ISO_URL = "http://mc0n1-srt.srt.mirantis.net/%s" % ADMIN_ISO_NAME

4
samples/node_ip_address_integration.py
View File

@@ -1,6 +1,6 @@
import unittest
import devops
from devops.helpers import wait, TimeoutError
from src import devops
from src.devops.helpers import wait, TimeoutError
import time
ISO_URL = 'http://mc0n1-srt.srt.mirantis.net/livecd.iso'

4
samples/run.py
View File

@@ -1,7 +1,7 @@
import time
import devops
from devops.helpers import wait, tcp_ping
from src import devops
from src.devops.helpers import wait, tcp_ping
import logging

7
setup.py
View File

@@ -2,12 +2,11 @@ from setuptools import setup
setup(
name='devops',
version='0.1',
description=
'Library to aid creating and manipulating virtual environments',
version='2.0',
description='Library for creating and manipulating virtual environments',
author='Mirantis, Inc.',
author_email='product@mirantis.com',
packages=['devops', 'devops.driver'],
scripts=['bin/devops'],
install_requires=['xmlbuilder', "ipaddr", "paramiko", "lxml", "pyyaml"]
install_requires=['xmlbuilder', "ipaddr", "paramiko", "lxml"]
)

1
src/__init__.py Normal file
View File

@@ -0,0 +1 @@
__author__ = 'vic'

34
devops/__main__.py → src/devops.py
View File

@@ -1,4 +1,6 @@
from main import *
__author__ = 'vic'
#!/usr/bin/env python
import os, sys
def saved(args):
@@ -31,3 +33,33 @@ elif arguments.command == 'resume':
else:
help()
sys.exit(1)
from src.devops.controller import Controller
class ControllerSingleton(Controller):
_instance = None
def __new__(cls, *args, **kwargs):
if not cls._instance:
cls._instance = super(ControllerSingleton, cls).__new__(
cls, *args, **kwargs)
return cls._instance
def getController():
return ControllerSingleton(Libvirt())
def build(environment):
getController().build_environment(environment)
def destroy(environment):
getController().destroy_environment(environment)
if __name__ == "__main__":
pass

0
devops/driver/__init__.py → src/devops/__init__.py
View File

72
src/devops/controller.py Normal file
View File

@@ -0,0 +1,72 @@
from src.devops.helpers.network import IpNetworksPool
from src.devops.error import DevopsError
import logging
logger = logging.getLogger(__name__)
class Controller:
def define_networks(self):
networks_pool = IpNetworksPool()
for network in environment.networks:
network.ip_addresses = self.networks_pool.get()
# if network.pxe:
# network.dhcp_server = True
# network.tftp_path
if network.dhcp_server or network.reserve_static:
allocated_addresses = []
for interface in network.interfaces:
for address in interface.ip_addresses:
if address in network.ip_addresses:
allocated_addresses.append(address)
dhcp_allowed_addresses = list(network.ip_addresses)[2:-2]
for interface in network.interfaces:
logger.info("Enumerated interfaces '%s' '%s'" % (interface.node, interface.network.name))
logger.info(list(interface.ip_addresses))
if not len(list(interface.ip_addresses)):
address = self.get_first_free_address(
dhcp_allowed_addresses,
allocated_addresses)
interface.ip_addresses.append(address)
allocated_addresses.append(address)
def build_environment(self, environment):
for network in environment.networks:
self.driver.create_network(network)
network.start()
for node in environment.nodes:
self._build_node(node)
node.driver = self.driver
def destroy_environment(self, environment):
for node in environment.nodes:
self.driver.stop_node(node)
for snapshot in node.snapshots:
self.driver.delete_snapshot(node, snapshot)
for disk in node.disks:
self.driver.delete_disk(disk)
self.driver.delete_node(node)
for network in environment.networks:
self.driver.delete_network(network)
def _build_node(self, node):
for disk in filter(lambda d: d.path is None, node.disks):
logger.debug("Creating disk file for node '%s'" % node.name)
disk.path = self.driver.create_disk(disk)
logger.debug("Creating node '%s'" % node.name)
self.driver.create_node(node)
def get_first_free_address(self, allowed_addresses, allocated_addresses):
s = set(allocated_addresses)
for x in allowed_addresses:
if x not in s:
return x
raise DevopsError("Free address not found")

0
src/devops/driver/__init__.py Normal file
View File

3
devops/driver/libvirt.py → src/devops/driver/libvirt-command-line.py
View File

@@ -1,5 +1,4 @@
# vim: ts=4 sw=4 expandtab
from devops import xml, scancodes
import os
import tempfile
import time
@@ -11,7 +10,7 @@ import ipaddr
import re
import logging
from devops.helpers import retry
from src.devops.helpers import retry, scancodes, xml
logger = logging.getLogger('devops.libvirt')

1
src/devops/driver/libvirt/__init__.py Normal file
View File

@@ -0,0 +1 @@
__author__ = 'vic'

163
src/devops/driver/libvirt/libvirt_driver.py Normal file
View File

@@ -0,0 +1,163 @@
# vim: ts=4 sw=4 expandtab
from time import sleep
import libvirt
from src.devops.driver.libvirt.libvirt_xml_builder import LibvirtXMLBuilder
from src.devops.helpers import scancodes
from src.devops.helpers.retry import retry
import xml.etree.ElementTree as ET
import ipaddr
from src.devops.models import Node
class LibvirtException(Exception):
pass
class LibvirtDriver:
def __init__(self, name, xml_builder=LibvirtXMLBuilder()):
self.xml_builder = xml_builder
libvirt.virInitialize()
self.conn = libvirt.open(name)
self.capabilities = None
def get_capabilities(self):
if self.capabilities is None:
self.capabilities = self.conn.getCapabilities()
return ET.fromstring(self.capabilities)
@retry()
def bridge_name(self, network):
self.conn.networkLookupByUUIDString(network.uuid).bridgeName()
@retry()
def create_network(self, network):
network.uuid = self.conn.networkDefineXML(
self.xml_builder.build_network_xml(network)
).UUID
@retry()
def delete_network(self, network):
self.conn.networkLookupByUUID(network.uuid).destroy()
self.conn.networkLookupByUUID(network.uuid).undefine()
@retry()
def start_network(self, network):
self.conn.networkLookupByUUIDString(network.uuid).create()
@retry()
def stop_network(self, network):
self.conn.networkLookupByUUIDString(network.uuid).destroy()
def create_node(self, node):
"""
:rtype : None
:type node: Node
"""
emulator = self.get_capabilities(
).find(
'guest/arch[@name="{0:>s}"]/domain[@type="{1:>s}"]/emulator'.format(
node.architecture, node.hypervisor)).text
node_xml = self.xml_builder.build_node_xml(node, emulator)
self.uuid = self.conn.createXML(node_xml, 0).UUID()
def delete_node(self, node):
self.conn.lookupByUUID(node.uuid).destroy()
self.conn.lookupByUUID(node.uuid).undefine()
def get_vnc_port(self, node):
xml_desc = ET.fromstring(self.conn.lookupByUUID(node.uuid).XMLDesc(0))
vnc_element = xml_desc.find('devices/graphics[@type="vnc"][@port]')
if vnc_element:
return vnc_element.get('port')
def start_node(self, node):
self.conn.lookupByUUID(node.uuid).create()
def stop_node(self, node):
self.destroy_node(node)
def reset_node(self, node):
self.conn.lookupByUUID(node.uuid).reset()
def reboot_node(self, node):
self.conn.lookupByUUID(node.uuid).reboot()
def suspend_node(self, node):
self.conn.lookupByUUID(node.uuid).suspend()
def resume_node(self, node):
self.conn.lookupByUUID(node.uuid).resume()
def shutdown_node(self, node):
self.conn.lookupByUUID(node.uuid).shutdown()
def destroy_node(self, node):
self.conn.lookupByUUID(node.uuid).destroy()
def get_node_snapshots(self, node):
self.conn.lookupByUUID(node.uuid).snapshotListNames(0)
def create_snapshot(self, node, name=None, description=None):
xml =self.xml_builder.build_snapshot_xml(name, description)
self.conn.lookupByUUID(node.uuid).snapshotCreateXML(xml)
def _get_snapshot(self, domain, name):
if name is None:
return domain.snapshotCurrent()
else:
return domain.snapshotLookupByName(name, 0)
def revert_snapshot(self, node, name=None):
domain = self.conn.lookupByUUID(node.uuid)
snapshot = self._get_snapshot(domain, name)
domain.revertToSnapshot(snapshot,0)
def delete_snapshot(self, node, name=None):
domain = self.conn.lookupByUUID(node.uuid)
snapshot = self._get_snapshot(domain, name)
snapshot.delete(0)
def send_keys_to_node(self, node, keys):
keys = scancodes.from_string(str(keys))
for key_codes in keys:
if isinstance(key_codes[0], str):
if key_codes[0] == 'wait':
sleep(1)
continue
self.conn.lookupByUUID(node.uuid).sendKey(0, 0, key_codes, len(key_codes), 0, 0)
def create_volume(self, volume, libvirt_pool='default'):
libvirt_pool = self.conn.storagePoolLookupByName(libvirt_pool)
libvirt_volume = libvirt_pool.createXML(self.xml_builder.build_volume_xml(volume))
volume.uuid = libvirt_volume.key()
def _get_file_size(self, file):
current = file.tell()
try:
file.seek(0, 2)
size = file.tell()
finally:
file.seek(current)
return size
def upload_volume(self, volume, path):
with open(path, 'rb') as f:
self.conn.storageVolLookupByKey(volume.uuid).upload(
stream = f, offset = 0,
length = self._get_file_size(f), flags = 0)
def delete_disk(self, disk):
self.conn.storageVolLookupByKey(disk.uuid)
def get_allocated_networks(self):
allocated_networks =[]
for network_name in self.conn.listDefinedNetworks():
et = ET.fromstring(self.conn.networkLookupByName(network_name).XMLDesc())
ip = et.find('ip[@address]')
if ip:
address = ip.get('address')
prefix_or_netmask = ip.get('prefix') or ip.get('netmask')
allocated_networks.append(ipaddr.IPNetwork(
"{0:>s}/{1:>s}".format(address, prefix_or_netmask)))
return allocated_networks

141
src/devops/driver/libvirt/libvirt_xml_builder.py Normal file
View File

@@ -0,0 +1,141 @@
from collections import deque
from ipaddr import IPNetwork
from xmlbuilder import XMLBuilder
from src.devops.models import Node
class LibvirtXMLBuilder:
NAME_SIZE = 80
def get_name(self, *args):
name = '_'.join(list(args))
if len(name) > self.NAME_SIZE:
hash_str = str(hash(name))
name=hash_str+name[len(name)-self.NAME_SIZE+len(hash_str):]
return name
def find(self, p, seq):
for item in seq:
if p(item):
return item
return None
def build_network_xml(self, network):
"""
:type network: Network
:rtype : String
"""
network_xml = XMLBuilder('network')
network_xml.name(self.get_name(network.environment.name, network.name))
if not (network.forward is None):
network_xml.forward(mode=network.forward)
if not (network.ip_network is None):
ip_network = IPNetwork(network.ip_network)
with network_xml.ip(
address=str(ip_network.network),
prefix=str(network.ip_network.prefixlen)):
if network.has_pxe_server:
network_xml.tftp(root=network.tftp_root_dir)
if network.has_dhcp_server:
with network_xml.dhcp:
allowed_addresses = list(network.ip_addresses)[2: - 2]
network_xml.range(start=str(start), end=str(end))
for interface in network.interfaces:
address = self.find(
lambda ip: ip in allowed_addresses,
interface.ip_addresses
)
if address and interface.mac_address:
network_xml.host(
mac=str(interface.mac_address),
ip=str(address),
name=interface.node.name
)
if network.pxe:
network_xml.bootp(file="pxelinux.0")
return str(network_xml)
def build_volume_xml(self, volume, backing_store_path=None):
"""
:type volume: Volume
:type backing_store_path: String
:rtype : String
"""
volume_xml = XMLBuilder('volume')
volume_xml.name(volume.name)
volume_xml.capacity(volume.capacity)
with volume_xml.target:
volume_xml.format(type=volume.format)
if volume.backing_store:
with volume_xml.backing_store:
volume_xml.path = backing_store_path
volume_xml.format = volume.backing_store.format
return str(volume_xml)
def build_snapshot_xml(self, name=None, description=None):
"""
:rtype : String
:type name: String
:type description: String
"""
xml_builder = XMLBuilder('domainsnapshot')
if not (name is None):
xml_builder.name(name)
if not (description is None):
xml_builder.description(description)
def build_node_xml(self, node, emulator):
"""
:rtype : String
:type node: Node
"""
node_xml = XMLBuilder("domain", type=node.hypervisor)
node_xml.name(node.name)
node_xml.vcpu(str(node.vcpu))
node_xml.memory(str(node.memory), unit='MiB')
with node_xml.os:
node_xml.type(node.os_type, arch=node.architecture)
for boot_dev in node.boot:
node_xml.boot(dev=boot_dev)
serial_disk_names = deque(
['sd' + c for c in list('abcdefghijklmnopqrstuvwxyz')])
def disk_name():
return serial_disk_names.popleft()
with node_xml.devices:
node_xml.emulator(emulator)
if len(node.disks) > 0:
node_xml.controller(type="ide")
for disk in node.disks:
with node_xml.disk(type="file", device="disk"):
# node_xml.driver(
# name="qemu", type=disk.format,
# cache="unsafe")
node_xml.source(file=disk.path)
node_xml.target(dev=disk_name(disk.bus), bus=disk.bus)
if node.cdrom:
with node_xml.disk(type="file", device="cdrom"):
# node_xml.driver(name="qemu", type="raw")
node_xml.source(file=node.cdrom.isopath)
node_xml.target(
dev=disk_name(node.cdrom.bus),
bus=node.cdrom.bus)
for interface in node.interfaces:
with node_xml.interface(type=interface.type):
node_xml.source(network=interface.network.id)
if not (interface.type is None):
node_xml.model(type=interface.type)
if node.has_vnc:
node_xml.graphics(type='vnc', listen='0.0.0.0', autoport='yes')
return str(node_xml)

0
devops/error.py → src/devops/error.py
View File

1
src/devops/helpers/__init__.py Normal file
View File

@@ -0,0 +1 @@
__author__ = 'vic'

17
devops/helpers.py → src/devops/helpers/helpers.py
View File

@@ -1,5 +1,4 @@
import os
import os.path
import urllib
import stat
import socket
@@ -7,7 +6,6 @@ import time
import httplib
import xmlrpclib
import paramiko
import string
import random
from threading import Thread
@@ -16,9 +14,9 @@ from SimpleHTTPServer import SimpleHTTPRequestHandler
import posixpath
import logging
from devops.error import DevopsError
from src.devops.error import DevopsError
logger = logging.getLogger('devops.helpers')
logger = logging.getLogger(__name__)
class TimeoutError(Exception):
@@ -309,14 +307,3 @@ def xmlrpcmethod(uri, method):
except:
raise AttributeError, "Error occured while getting server method"
def retry(count, func, **kwargs):
i = 0
while True:
#noinspection PyBroadException
try:
return func(**kwargs)
except:
if i >= count:
raise
i += 1
time.sleep(1)

21
src/devops/helpers/network.py Normal file
View File

@@ -0,0 +1,21 @@
import ipaddr
class IpNetworksPool:
def __init__(self, networks=None, prefix=24):
if networks is None:
networks = [ipaddr.IPNetwork('10.0.0.0/16')]
self.networks = networks
self.prefix = prefix
def overlaps(self, network, allocated_networks):
for allocated_network in allocated_networks:
if allocated_network.overlaps(network):
return True
return False
def get(self, allocated_networks):
for network in self.networks:
for sub_net in network.iter_subnets:
if not self.overlaps(sub_net, allocated_networks):
yield sub_net

19
src/devops/helpers/retry.py Normal file
View File

@@ -0,0 +1,19 @@
import functools
from time import sleep
def retry(count=10, delay=1):
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
i = 0
while True:
#noinspection PyBroadException
try:
return func(*args, **kwargs)
except:
if i >= count:
raise
i += 1
sleep(delay)
return wrapper
return decorator

0
devops/scancodes.py → src/devops/helpers/scancodes.py
View File

5
devops/model.py → src/devops/model.py
View File

@@ -1,5 +1,4 @@
from itertools import chain
from devops.helpers import retry
from src.devops.helpers import retry
class EnvironmentException(object):
@@ -51,6 +50,8 @@ class Environment(ManagedObject):
return name2network
class Network(ManagedObject):
def __init__(self, name, dhcp_server=False, pxe=False,
reserve_static=True, forward='nat'):

119
src/devops/models.py Normal file
View File

@@ -0,0 +1,119 @@
__author__ = 'vic'
from django.db import models
class EnvironmentManager(models.Manager):
def create(self, name):
return super(EnvironmentManager, self).create(name=name)
def get_or_create(self, name):
return super(EnvironmentManager, self).get_or_create(name=name)
class Environment(models.Model):
name = models.CharField(max_length=255, unique=True, null=False)
objects = EnvironmentManager()
class NetworkManager(models.Manager):
# def create(self, name, dhcp_server=False, pxe=False, reserve_static=True, forward='nat'):
# return super(NetworkManager, self).create(name=name)
pass
class Network(models.Model):
FORWARD_MODE_CHOOSES = (
('nat','nat'),
('route','route'),
('bridge','bridge'),
('private','private'),
('vepa','vepa'),
('passthrough','passthrough'),
('hostdev','hostdev'),
)
name = models.CharField(max_length=255, unique=True, null=False)
uuid = models.CharField(max_length=255)
has_dhcp_server = models.BooleanField(default=False, null=False)
has_tftp_server = models.BooleanField(default=False, null=False)
has_reserved_ips = models.BooleanField(default=True, null=False)
tftp_root_dir = models.CharField(max_length=255, null=True)
forward = models.CharField(max_length=10, null=False, choises=FORWARD_MODE_CHOOSES)
ip_network = models.CharField(max_length=255, unique=True, null=True)
environment = models.ForeignKey(Environment, null=True)
objects = NetworkManager()
class NodeManager(models.Manager):
# def create(self, name, cpu=1, memory=512, arch='x86_64', vnc=False,
# metadata=None):
# super(NodeManager, self).create()
#if metadata is None:
# self.metadata = {}
#else:
#self.metadata = metadata
#
#
pass
class Node(models.Model):
ARCHITECTURE_CHOOSES = (
('nat','nat'),
('route','route'),
('bridge','bridge'),
('private','private'),
('vepa','vepa'),
('passthrough','passthrough'),
('hostdev','hostdev'),
)
hypervisor = 'kvm'
os_type = 'hvm'
architecture = 'x86_64'
boot = 'network'
name = models.CharField(max_length=255, unique=True, null=False)
uuid = models.CharField(max_length=255)
# metadata = models.CharField(max_length=255, null=False, default='{}')
vcpu = models.IntegerField(null=False, default=1)
memory = models.IntegerField(null=False, default=1024)
vnc = models.BooleanField(null=False, default=True)
environment = models.ForeignKey(Environment, null=True)
objects = NodeManager()
# arch = models.CharField(max_length=10, null=False, choises)
# self.interfaces = []
# self.bridged_interfaces = []
# self.disks = []
# self.boot = []
# self.cdrom = None
# self.environment = None
class VolumeManager(models.Manager):
# def __init__(self, capacity=None, path=None, format='qcow2', base_image=None):
pass
def upload(self, path):
pass
class Volume(models.Model):
name = models.CharField(max_length=255, unique=True, null=False)
uuid = models.CharField(max_length=255)
capacity = models.IntegerField(null=False)
backing_store = models.ForeignKey('self', null=True)
format = models.CharField(max_length=255, null=False)
bus = models.CharField(max_length=255)
objects = VolumeManager()
environment = models.ForeignKey(Environment, null=True)
class InterfaceManager(models.Manager):
pass
class Interface(models.Model):
name = models.CharField(max_length=255, unique=True, null=False)
uuid = models.CharField(max_length=255)
mac_address = models.CharField(max_length=255, unique=True, null=False)
network = models.ForeignKey(Network)
node = models.ForeignKey(Node)
type = models.CharField(max_length=255, null=False)
target_dev = models.CharField(max_length=255, unique=True, null=False)
class AddressManager(models.Manager):
pass
class Address(models.Model):
ip_address = models.GenericIPAddressField()
interface = models.ForeignKey(Interface)
objects = AddressManager()

9
src/manage.py Normal file
View File

@@ -0,0 +1,9 @@
#!/usr/bin/env python
import os, sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "devops.settings")
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)

1
src/settings/__init__.py Normal file
View File

@@ -0,0 +1 @@
__author__ = 'vic'

8
src/settings/settings.py Normal file
View File

@@ -0,0 +1,8 @@
import os
__author__ = 'vic'
DRIVER = 'src.devops.driver.libvirt.libvirt_driver.LibvirtDriver'
HOME_DIR = os.environ.get('DEVOPS_HOME') or os.environ.get('APPDATA') or os.environ['HOME']

1
src/test/__init__.py Normal file
View File

@@ -0,0 +1 @@
__author__ = 'vic'

8
test/test_libvirt.py → src/test/test_libvirt.py
View File

@@ -1,10 +1,10 @@
import unittest
import re
from ipaddr import IPv4Network
from devops import xml
from devops.model import Network, Node, Disk, Cdrom
from devops.network import IPv4Address, IPv4Network
from devops.driver.libvirt import Libvirt, LibvirtXMLBuilder, DeploymentSpec
from src.devops.helpers import xml
from src.devops.model import Network, Node, Cdrom
from src.devops.helpers.network import IPv4Address
from src.devops.driver.libvirt import Libvirt, LibvirtXMLBuilder, DeploymentSpec
class TestLibvirtXMLBuilder(unittest.TestCase):

10
test/test_network.py → src/test/test_network.py
View File

@@ -1,10 +1,10 @@
import unittest
from devops.network import IpNetworksPool, IPv4Network, NetworkPoolException
from src.devops.helpers.network import IpNetworksPool, IPv4Network, NetworkPoolException
class TestIpNetworksPool(unittest.TestCase):
def test_getting_subnetworks(self):
n = IpNetworksPool(net_addresses=['10.1.0.0/22'], prefix=24)
n = IpNetworksPool(networks=['10.1.0.0/22'], prefix=24)
nets = []
while not n.is_empty:
@@ -17,7 +17,7 @@ class TestIpNetworksPool(unittest.TestCase):
self.assertTrue(IPv4Network('10.1.3.0/24') in nets)
def test_putting_back(self):
n = IpNetworksPool(net_addresses=['10.1.0.0/22'], prefix=24)
n = IpNetworksPool(networks=['10.1.0.0/22'], prefix=24)
while not n.is_empty:
n.get()
self.assertTrue(n.is_empty)
@@ -27,14 +27,14 @@ class TestIpNetworksPool(unittest.TestCase):
self.assertTrue(n.is_empty)
def test_putting_back_network_not_from_this_pool_raises_error(self):
n = IpNetworksPool(net_addresses=['10.1.0.0/22'], prefix=24)
n = IpNetworksPool(networks=['10.1.0.0/22'], prefix=24)
while not n.is_empty:
n.get()
with self.assertRaises(NetworkPoolException):
n.put(IPv4Network('10.2.1.0/24'))
def test_reserve(self):
n = IpNetworksPool(net_addresses=['10.0.0.0/22'], prefix=24)
n = IpNetworksPool(networks=['10.0.0.0/22'], prefix=24)
n.reserve(IPv4Network('10.0.0.0/23'))
nets = []
while not n.is_empty:

2
test/test_scancodes.py → src/test/test_scancodes.py
View File

@@ -1,5 +1,5 @@
import unittest
from devops import scancodes
from src.devops.helpers import scancodes
class TestScancodes(unittest.TestCase):

2
test/test_xml.py → src/test/test_xml.py
View File

@@ -1,5 +1,5 @@
from devops import xml
import unittest
from src.devops.helpers import xml
class TestXml(unittest.TestCase):

2
test/test_yaml_config_loader.py → src/test/test_yaml_config_loader.py
View File

@@ -1,5 +1,5 @@
import unittest
from devops import yaml_config_loader
from src.devops import yaml_config_loader
class TestYamlConfigLoader(unittest.TestCase):