openstack/fuel-devops
Code Issues Proposed changes

Refornat code

Browse Source
This commit is contained in:
vic
2013-01-22 19:57:50 +04:00
parent eb9e3164a6
commit 51bf0dc460
12 changed files with 150 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
samples/one.py
View File

@@ -2,25 +2,32 @@ import ipaddr
def one(manager):
environment = manager.environment_create('test_env7')
internal_pool = manager.create_network_pool(networks=[ipaddr.IPNetwork('10.108.0.0/16')], prefix=24
internal_pool = manager.create_network_pool(
networks=[ipaddr.IPNetwork('10.108.0.0/16')], prefix=24
)
private_pool = manager.create_network_pool(networks=[ipaddr.IPNetwork('10.108.0.0/16')], prefix=24
private_pool = manager.create_network_pool(
networks=[ipaddr.IPNetwork('10.108.0.0/16')], prefix=24
)
external_pool = manager.create_network_pool(networks=[ipaddr.IPNetwork('172.18.95.0/24')], prefix=27
external_pool = manager.create_network_pool(
networks=[ipaddr.IPNetwork('172.18.95.0/24')], prefix=27
)
internal = manager.network_create(
environment=environment, name='internal', pool=internal_pool)
external = manager.network_create(
environment=environment, name='external', pool=external_pool, forward='nat')
environment=environment, name='external', pool=external_pool,
forward='nat')
private = manager.network_create(
environment=environment, name='private', pool=private_pool)
node = manager.node_create(name='test_node', environment=environment)
manager.interface_create(node=node, network=internal)
manager.interface_create(node=node, network=external)
manager.interface_create(node=node, network=private)
volume = manager.volume_get_predefined('/var/lib/libvirt/images/centos63-cobbler-base.qcow2')
v3 = manager.volume_create_child('test_vp895', backing_store=volume, environment=environment)
v4 = manager.volume_create_child('test_vp896', backing_store=volume, environment=environment)
volume = manager.volume_get_predefined(
'/var/lib/libvirt/images/centos63-cobbler-base.qcow2')
v3 = manager.volume_create_child('test_vp895', backing_store=volume,
environment=environment)
v4 = manager.volume_create_child('test_vp896', backing_store=volume,
environment=environment)
manager.node_attach_volume(node=node, volume=v3)
manager.node_attach_volume(node, v4)
environment.define()

3
setup.py
View File

@@ -6,7 +6,8 @@ setup(
description='Library for creating and manipulating virtual environments',
author='Mirantis, Inc.',
author_email='product@mirantis.com',
packages=['devops', 'devops.driver', 'devops.helpers', 'devops.tests', 'devops.driver.libvirt'],
packages=['devops', 'devops.driver', 'devops.helpers', 'devops.tests',
'devops.driver.libvirt'],
package_dir={'': 'src'},
scripts=['src/dos.py'],
install_requires=['xmlbuilder', "ipaddr", "paramiko", "django"]

9
src/devops/driver/libvirt/libvirt_driver.py
View File

@@ -105,7 +105,8 @@ class LibvirtDriver(object):
"""
:rtype : None
"""
ret = self.conn.networkDefineXML(self.xml_builder.build_network_xml(network))
ret = self.conn.networkDefineXML(
self.xml_builder.build_network_xml(network))
ret.setAutostart(True)
network.uuid = ret.UUIDString()
@@ -173,7 +174,8 @@ class LibvirtDriver(object):
:type node: Node
:rtype : String
"""
xml_desc = ET.fromstring(self.conn.lookupByUUIDString(node.uuid).XMLDesc(0))
xml_desc = ET.fromstring(
self.conn.lookupByUUIDString(node.uuid).XMLDesc(0))
vnc_element = xml_desc.find('devices/graphics[@type="vnc"][@port]')
if vnc_element:
return vnc_element.get('port')
@@ -358,7 +360,8 @@ class LibvirtDriver(object):
:type volume: Volume
:rtype : String
"""
xml_desc = ET.fromstring(self.conn.storageVolLookupByKey(volume.uuid).XMLDesc(0))
xml_desc = ET.fromstring(
self.conn.storageVolLookupByKey(volume.uuid).XMLDesc(0))
return xml_desc.find('target/format[@type]').get('type')
@retry()

18
src/devops/driver/libvirt/libvirt_xml_builder.py
View File

@@ -22,7 +22,9 @@ class LibvirtXMLBuilder(object):
:rtype : String
"""
network_xml = XMLBuilder('network')
network_xml.name(self._get_name(network.environment and network.environment.name or '', network.name))
network_xml.name(self._get_name(
network.environment and network.environment.name or '',
network.name))
if not (network.forward is None):
network_xml.forward(mode=network.forward)
if not (network.ip_network is None):
@@ -34,7 +36,8 @@ class LibvirtXMLBuilder(object):
network_xml.tftp(root=network.tftp_root_dir)
if network.has_dhcp_server:
with network_xml.dhcp:
network_xml.range(start=str(network.ip_pool_start), end=str(network.ip_pool_end))
network_xml.range(start=str(network.ip_pool_start),
end=str(network.ip_pool_end))
for interface in network.interfaces:
for address in interface.addresses:
if IPAddress(address.ip_address) in ip_network:
@@ -54,7 +57,9 @@ class LibvirtXMLBuilder(object):
:rtype : String
"""
volume_xml = XMLBuilder('volume')
volume_xml.name(self._get_name(volume.environment and volume.environment.name or '', volume.name))
volume_xml.name(
self._get_name(volume.environment and volume.environment.name or '',
volume.name))
volume_xml.capacity(str(volume.capacity))
with volume_xml.target:
volume_xml.format(type=volume.format)
@@ -88,7 +93,8 @@ class LibvirtXMLBuilder(object):
raise NotImplementedError()
with device_xml.interface(type=interface.type):
device_xml.mac(address=interface.mac_address)
device_xml.source(network=self.driver.network_name(interface.network))
device_xml.source(
network=self.driver.network_name(interface.network))
if not (interface.type is None):
device_xml.model(type=interface.model)
@@ -98,7 +104,9 @@ class LibvirtXMLBuilder(object):
:type node: Node
"""
node_xml = XMLBuilder("domain", type=node.hypervisor)
node_xml.name(self._get_name(node.environment and node.environment.name or '', node.name))
node_xml.name(
self._get_name(node.environment and node.environment.name or '',
node.name))
node_xml.vcpu(str(node.vcpu))
node_xml.memory(str(node.memory), unit='MiB')

17
src/devops/error.py
View File

@@ -1,13 +1,24 @@
class DevopsError(Exception):
message = "Devops Error"
class DevopsCalledProcessError(DevopsError):
def __init__(self, returncode, cmd, output=None):
self.returncode = returncode
self.cmd = cmd
self.output = output
def __str__(self):
message = "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
message = "Command '%s' returned non-zero exit status %d" % (
self.cmd, self.returncode)
if self.output:
message+="\n%s" % '\n'.join(self.output)
return message
message += "\n%s" % '\n'.join(self.output)
return message
class TimeoutError(DevopsError):
pass
class AuthenticationError(DevopsError):
pass

36
src/devops/helpers/helpers.py
View File

@@ -1,4 +1,3 @@
import subprocess
import os
import urllib
import stat
@@ -9,43 +8,37 @@ import xmlrpclib
import paramiko
import random
from threading import Thread
import BaseHTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler
import posixpath
import logging
from devops.error import DevopsError, DevopsCalledProcessError
from devops.error import DevopsError, DevopsCalledProcessError, TimeoutError, AuthenticationError
logger = logging.getLogger(__name__)
class TimeoutError(Exception):
pass
class AuthenticationError(Exception):
pass
def get_free_port():
ports = range(32000, 32100)
random.shuffle(ports)
for port in ports:
if not tcp_ping('localhost', port):
return port
raise DevopsError, "No free ports available"
raise DevopsError("No free ports available")
def icmp_ping(host, timeout=1):
"icmp_ping(host, timeout=1) - returns True if host is pingable; False - otherwise."
"""
icmp_ping(host, timeout=1) - returns True if host is pingable; False - otherwise.
"""
return os.system(
"ping -c 1 -W '%(timeout)d' '%(host)s' 1>/dev/null 2>&1" % {
'host': str(host), 'timeout': timeout}) == 0
def tcp_ping(host, port):
"tcp_ping(host, port) - returns True if TCP connection to specified host and port can be established; False - otherwise."
"""
tcp_ping(host, port) - returns True if TCP connection to specified host and port can be established; False - otherwise.
"""
s = socket.socket()
try:
s.connect((str(host), int(port)))
@@ -65,7 +58,7 @@ def wait(predicate, interval=5, timeout=None):
start_time = time.time()
while not predicate():
if timeout and start_time + timeout < time.time():
raise TimeoutError, "Waiting timed out"
raise TimeoutError("Waiting timed out")
seconds_to_sleep = interval
if timeout:
@@ -143,7 +136,7 @@ class SSHClient(object):
def check_stderr(self, command, verbose=False):
ret = self.check_call(command, verbose)
if ret['stderr'] :
if ret['stderr']:
raise DevopsCalledProcessError(command, ret, ret['stderr'])
return ret
@@ -317,7 +310,7 @@ def xmlrpctoken(uri, login, password):
try:
return server.login(login, password)
except:
raise AuthenticationError, "Error occured while login process"
raise AuthenticationError("Error occured while login process")
def xmlrpcmethod(uri, method):
@@ -325,10 +318,9 @@ def xmlrpcmethod(uri, method):
try:
return getattr(server, method)
except:
raise AttributeError, "Error occured while getting server method"
raise AttributeError("Error occured while getting server method")
def generate_mac():
return "64:{0:02x}:{1:02x}:{2:02x}:{3:02x}:{4:02x}".format(*bytearray(os.urandom(5)))
return "64:{0:02x}:{1:02x}:{2:02x}:{3:02x}:{4:02x}".format(
*bytearray(os.urandom(5)))

1
src/devops/helpers/retry.py
View File

@@ -1,6 +1,7 @@
import functools
from time import sleep
def retry(count=10, delay=1):
def decorator(func):
@functools.wraps(func)

57
src/devops/manager.py
View File

@@ -6,6 +6,7 @@ from devops.helpers.helpers import generate_mac
from devops.helpers.network import IpNetworksPool
from devops.models import Address, Interface, Node, Network, Environment, Volume, DiskDevice, ExternalModel
class Manager(object):
def __init__(self):
super(Manager, self).__init__()
@@ -26,23 +27,34 @@ class Manager(object):
return pool
def _get_default_pool(self):
self.default_pool = self.default_pool or self.create_network_pool(networks=[ipaddr.IPNetwork('10.0.0.0/16')],
self.default_pool = self.default_pool or self.create_network_pool(
networks=[ipaddr.IPNetwork('10.0.0.0/16')],
prefix=24)
return self.default_pool
def network_create(
self, name, environment=None, ip_network=None, pool=None, has_dhcp_server=True, has_pxe_server=False,
forward='nat'):
allocated_network = ip_network or ExternalModel.allocate_network(pool or self._get_default_pool())
return Network.objects.create(environment=environment, name=name, ip_network=ip_network or allocated_network,
has_pxe_server=has_pxe_server, has_dhcp_server=has_dhcp_server, forward=forward)
self, name, environment=None, ip_network=None, pool=None,
has_dhcp_server=True, has_pxe_server=False,
forward='nat'
):
allocated_network = ip_network or ExternalModel.allocate_network(
pool or self._get_default_pool())
return Network.objects.create(
environment=environment, name=name,
ip_network=ip_network or allocated_network,
has_pxe_server=has_pxe_server, has_dhcp_server=has_dhcp_server,
forward=forward)
def node_create(self, name, environment=None, role=None, vcpu=2,
memory=1024, has_vnc=True, metadata=None, hypervisor='kvm',
os_type='hvm', architecture='x86_64', boot=None):
if not boot: boot = ['network', 'cdrom', 'hd']
node = Node.objects.create(name=name, environment=environment, role=role, vcpu=vcpu, memory=memory,
has_vnc=has_vnc, metadata=metadata, hypervisor=hypervisor, os_type=os_type, architecture=architecture,
if not boot:
boot = ['network', 'cdrom', 'hd']
node = Node.objects.create(
name=name, environment=environment,
role=role, vcpu=vcpu, memory=memory,
has_vnc=has_vnc, metadata=metadata, hypervisor=hypervisor,
os_type=os_type, architecture=architecture,
)
node.boot = boot
return node
@@ -56,13 +68,17 @@ class Manager(object):
volume.save()
return volume
def volume_create_child(self, name, backing_store, format=None, environment=None):
return Volume.objects.create(name=name, environment=environment, capacity=backing_store.capacity,
def volume_create_child(self, name, backing_store, format=None,
environment=None):
return Volume.objects.create(
name=name, environment=environment,
capacity=backing_store.capacity,
format=format or backing_store.format, backing_store=backing_store)
def volume_create(self, name, capacity, format='qcow2', environment=None):
return Volume.objects.create(name=name, environment=environment, capacity=capacity, format=format)
return Volume.objects.create(
name=name, environment=environment,
capacity=capacity, format=format)
def upload(self, path):
pass
@@ -70,8 +86,11 @@ class Manager(object):
def _generate_mac(self):
return generate_mac()
def interface_create(self, network, node, type='network', target_dev=None, mac_address=None, model='virtio'):
interface = Interface.objects.create(network=network, node=node, type=type, target_dev=target_dev,
def interface_create(self, network, node, type='network', target_dev=None,
mac_address=None, model='virtio'):
interface = Interface.objects.create(
network=network, node=node,
type=type, target_dev=target_dev,
mac_address=mac_address or self._generate_mac(), model=model)
interface.add_address(str(network.next_ip()))
return interface
@@ -79,7 +98,9 @@ class Manager(object):
def network_create_address(self, ip_address, interface):
Address.objects.create(ip_address=ip_address, interface=interface)
def node_attach_volume(self, node, volume, device='disk', type='file', bus='virtio', target_dev=None):
DiskDevice.objects.create(device=device, type=type, bus=bus, target_dev=target_dev or node.next_disk_name(),
def node_attach_volume(self, node, volume, device='disk', type='file',
bus='virtio', target_dev=None):
DiskDevice.objects.create(
device=device, type=type, bus=bus,
target_dev=target_dev or node.next_disk_name(),
volume=volume, node=node)

12
src/devops/models.py
View File

@@ -2,6 +2,7 @@ from ipaddr import IPNetwork
from devops.driver.libvirt.libvirt_driver import LibvirtDriver
from django.db import models
def choices(*args, **kwargs):
defaults = {'max_length': 255, 'null': False}
defaults.update(kwargs)
@@ -128,7 +129,9 @@ class Network(ExternalModel):
has_pxe_server = models.BooleanField()
has_reserved_ips = models.BooleanField(default=True)
tftp_root_dir = models.CharField(max_length=255)
forward = choices('nat', 'route', 'bridge', 'private', 'vepa', 'passthrough', 'hostdev')
forward = choices(
'nat', 'route', 'bridge', 'private', 'vepa',
'passthrough', 'hostdev')
ip_network = models.CharField(max_length=255, unique=True)
@property
@@ -145,11 +148,14 @@ class Network(ExternalModel):
def next_ip(self):
while True:
self._iterhosts = self._iterhosts or IPNetwork(self.ip_network).iterhosts()
self._iterhosts = self._iterhosts or IPNetwork(
self.ip_network).iterhosts()
ip = self._iterhosts.next()
if ip < self.ip_pool_start or ip > self.ip_pool_end:
continue
if not Address.objects.filter(interface__network=self, ip_address=str(ip)).exists():
if not Address.objects.filter(
interface__network=self,
ip_address=str(ip)).exists():
return ip
def bridge_name(self):

3
src/devops/settings.py
View File

@@ -1,7 +1,8 @@
import os
DRIVER = 'src.devops.driver.libvirt.libvirt_driver.LibvirtDriver'
HOME_DIR = os.environ.get('DEVOPS_HOME') or os.environ.get('APPDATA') or os.environ['HOME']
HOME_DIR = os.environ.get('DEVOPS_HOME') or os.environ.get('APPDATA') or
os.environ['HOME']
INSTALLED_APPS = ['devops']
DATABASES = {

21
src/devops/shell.py
View File

@@ -37,10 +37,12 @@ class Shell(object):
self.manager.environment_get(self.params.name).resume()
def do_revert(self):
self.manager.environment_get(self.params.name).revert(self.params.snapshot_name)
self.manager.environment_get(self.params.name).revert(
self.params.snapshot_name)
def do_snapshot(self):
self.manager.environment_get(self.params.name).snapshot(self.params.snapshot_name)
self.manager.environment_get(self.params.name).snapshot(
self.params.snapshot_name)
commands = {
@@ -57,10 +59,13 @@ class Shell(object):
def get_params(self):
name_parser = argparse.ArgumentParser(add_help=False)
name_parser.add_argument('name', help='environment name', default=os.getenv('ENV_NAME'))
name_parser.add_argument('name', help='environment name',
default=os.getenv('ENV_NAME'))
snapshot_name_parser = argparse.ArgumentParser(add_help=False)
snapshot_name_parser.add_argument('--snapshot-name', help='snapshot name', default=os.getenv('SNAPSHOT_NAME'))
parser = argparse.ArgumentParser(description="Manage virtual environments")
snapshot_name_parser.add_argument('--snapshot-name',
help='snapshot name', default=os.getenv('SNAPSHOT_NAME'))
parser = argparse.ArgumentParser(
description="Manage virtual environments")
subparsers = parser.add_subparsers(help='commands', dest='command')
subparsers.add_parser('list')
subparsers.add_parser('show', parents=[name_parser])
@@ -69,6 +74,8 @@ class Shell(object):
subparsers.add_parser('destroy', parents=[name_parser])
subparsers.add_parser('suspend', parents=[name_parser])
subparsers.add_parser('resume', parents=[name_parser])
subparsers.add_parser('revert', parents=[name_parser, snapshot_name_parser])
subparsers.add_parser('snapshot', parents=[name_parser, snapshot_name_parser])
subparsers.add_parser('revert',
parents=[name_parser, snapshot_name_parser])
subparsers.add_parser('snapshot',
parents=[name_parser, snapshot_name_parser])
return parser.parse_args()

33
src/devops/tests/test_manager.py
View File

@@ -29,9 +29,11 @@ class TestManager(TestCase):
network = self.manager.network_create(
environment=environment, name='internal', ip_network='10.1.0.0/24')
interface = self.manager.interface_create(network=network, node=node)
self.manager.network_create_address(str('10.1.0.1'), interface=interface)
self.manager.network_create_address(str('10.1.0.1'),
interface=interface)
ip = network.next_ip()
self.manager.network_create_address(str('10.1.0.3'), interface=interface)
self.manager.network_create_address(str('10.1.0.3'),
interface=interface)
ip = network.next_ip()
self.assertEquals('10.1.0.4', str(ip))
@@ -58,7 +60,8 @@ class TestManager(TestCase):
def test_node_creationw(self):
environment = self.manager.environment_create('test_env55')
node = self.manager.node_create(name='test_node4', environment=environment)
node = self.manager.node_create(name='test_node4',
environment=environment)
node.define()
def test_node_creation(self):
@@ -69,7 +72,8 @@ class TestManager(TestCase):
# environment=environment, name='external', pool=None)
# private = self.manager.create_network(
# environment=environment, name='private', pool=None)
node = self.manager.node_create(name='test_node', environment=environment)
node = self.manager.node_create(name='test_node',
environment=environment)
self.manager.interface_create(node=node, network=internal)
# self.manager.create_interface(node=node, network=external)
# self.manager.create_interface(node=node, network=private)
@@ -77,12 +81,15 @@ class TestManager(TestCase):
def test_create_volume(self):
environment = self.manager.environment_create('test_env3')
volume = self.manager.volume_get_predefined('/var/lib/libvirt/images/disk-135824657433.qcow2')
v3 = self.manager.volume_create_child('test_vp89', backing_store=volume, environment=environment)
volume = self.manager.volume_get_predefined(
'/var/lib/libvirt/images/disk-135824657433.qcow2')
v3 = self.manager.volume_create_child('test_vp89', backing_store=volume,
environment=environment)
v3.define()
def test_create_volume(self):
volume = self.manager.volume_get_predefined('/var/lib/libvirt/images/disk-135824657433.qcow2')
volume = self.manager.volume_get_predefined(
'/var/lib/libvirt/images/disk-135824657433.qcow2')
v3 = self.manager.volume_create_child('test_vp89', backing_store=volume)
v3.define()
@@ -94,13 +101,17 @@ class TestManager(TestCase):
environment=environment, name='external', pool=None)
private = self.manager.network_create(
environment=environment, name='private', pool=None)
node = self.manager.node_create(name='test_node', environment=environment)
node = self.manager.node_create(name='test_node',
environment=environment)
self.manager.interface_create(node=node, network=internal)
self.manager.interface_create(node=node, network=external)
self.manager.interface_create(node=node, network=private)
volume = self.manager.volume_get_predefined('/var/lib/libvirt/images/disk-135824657433.qcow2')
v3 = self.manager.volume_create_child('test_vp892', backing_store=volume, environment=environment)
v4 = self.manager.volume_create_child('test_vp891', backing_store=volume, environment=environment)
volume = self.manager.volume_get_predefined(
'/var/lib/libvirt/images/disk-135824657433.qcow2')
v3 = self.manager.volume_create_child('test_vp892',
backing_store=volume, environment=environment)
v4 = self.manager.volume_create_child('test_vp891',
backing_store=volume, environment=environment)
self.manager.node_attach_volume(node=node, volume=v3)
self.manager.node_attach_volume(node, v4)
environment.define()