openstack/fuel-web
Code Issues Proposed changes

Modified fuel_agent_ci

Browse Source 
Change-Id: If371eb8d3e1dceae37ad5965c9ea8ca3e336fa94
Implements: blueprint image-based-provisioning
This commit is contained in:
Vladimir Kozhukalov 2014-07-29 11:05:24 +04:00
parent 1abf17806e
commit cce8d62485
23 changed files with 1329 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
fuel_agent_ci/fuel_agent_ci/cmd/ci.py
View File

@ -14,6 +14,8 @@
import argparse
import logging
import signal
import sys
import yaml
@ -24,33 +26,88 @@ logging.basicConfig(level=logging.DEBUG)
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument(
'-f', '--file', dest='env_file', action='store',
type=str, help='Environment data file', required=True
)
subparsers = parser.add_subparsers(dest='action')
create_parser = subparsers.add_parser('create')
create_parser.add_argument(
'-f', '--file', dest='env_file', action='store',
type=str, help='Environment data file', required=True
env_parser = subparsers.add_parser('env')
env_parser.add_argument(
'-a', '--action', dest='env_action', action='store',
type=str, help='Env action', required=True
)
env_parser.add_argument(
'-k', '--kwargs', dest='env_kwargs', action='store',
type=str, required=False,
help='Env action kwargs, must be valid json or yaml',
)
env_parser.add_argument(
'-K', '--kwargs_file', dest='env_kwargs_file', action='store',
type=str, required=False,
help='Env action kwargs file, content must be valid json or yaml',
)
destroy_parser = subparsers.add_parser('destroy')
destroy_parser.add_argument(
'-f', '--file', dest='env_file', action='store',
type=str, help='Environment data file', required=True
item_parser = subparsers.add_parser('item')
item_parser.add_argument(
'-t', '--type', dest='item_type', action='store',
type=str, help='Item type', required=True
)
item_parser.add_argument(
'-a', '--action', dest='item_action', action='store',
type=str, help='Item action', required=True
)
item_parser.add_argument(
'-n', '--name', dest='item_name', action='store',
type=str, help='Item name', required=False
)
item_parser.add_argument(
'-k', '--kwargs', dest='item_kwargs', action='store',
type=str, required=False,
help='Item action kwargs, must be valid json or yaml',
)
item_parser.add_argument(
'-K', '--kwargs_file', dest='item_kwargs_file', action='store',
type=str, required=False,
help='Item action kwargs file, content must be valid json or yaml',
)
return parser
def main():
def term_handler(signum=None, sigframe=None):
sys.exit()
signal.signal(signal.SIGTERM, term_handler)
signal.signal(signal.SIGINT, term_handler)
parser = parse_args()
params, other_params = parser.parse_known_args()
with open(params.env_file, "r") as f:
with open(params.env_file) as f:
env_data = yaml.load(f.read())
manager = ci_manager.Manager(env_data)
if params.action == 'create':
manager.define()
elif params.action == 'destroy':
manager.undefine()
# print 'params: %s' % params
# print 'other_params: %s' % other_params
if params.action == 'env':
kwargs = {}
if params.env_kwargs:
kwargs.update(yaml.load(params.env_kwargs))
elif params.env_kwargs_file:
with open(params.env_kwargs_file) as f:
kwargs.update(yaml.load(f.read()))
manager.do_env(params.env_action, **kwargs)
elif params.action == 'item':
kwargs = {}
if params.item_kwargs:
kwargs.update(yaml.load(params.item_kwargs))
elif params.item_kwargs_file:
with open(params.item_kwargs_file) as f:
kwargs.update(yaml.load(f.read()))
manager.do_item(params.item_type, params.item_action,
params.item_name, **kwargs)
if __name__ == '__main__':

52
fuel_agent_ci/fuel_agent_ci/drivers/__init__.py
View File

@ -11,3 +11,55 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from fuel_agent_ci.drivers import common_driver
from fuel_agent_ci.drivers import fabric_driver
from fuel_agent_ci.drivers import libvirt_driver
from fuel_agent_ci.drivers import pygit2_driver
from fuel_agent_ci.drivers import simple_http_driver
class Driver(object):
default_hierarchy = {
# these methods are from common_driver
'artifact_get': common_driver,
'artifact_clean': common_driver,
'artifact_status': common_driver,
# these methods are from fabric_driver
'ssh_status': fabric_driver,
'ssh_put_content': fabric_driver,
'ssh_put_file': fabric_driver,
'ssh_run': fabric_driver,
# these methods are from libvirt_driver
'net_start': libvirt_driver,
'net_stop': libvirt_driver,
'net_status': libvirt_driver,
'vm_start': libvirt_driver,
'vm_stop': libvirt_driver,
'vm_status': libvirt_driver,
'dhcp_start': libvirt_driver,
'dhcp_stop': libvirt_driver,
'dhcp_status': libvirt_driver,
'tftp_start': libvirt_driver,
'tftp_stop': libvirt_driver,
'tftp_status': libvirt_driver,
# these methods are from pygit2_driver
'repo_clone': pygit2_driver,
'repo_clean': pygit2_driver,
'repo_status': pygit2_driver,
# these methods are from simple_http_driver
'http_start': simple_http_driver,
'http_stop': simple_http_driver,
'http_status': simple_http_driver,
}
def __init__(self, hierarchy=None):
self.hierarchy = self.default_hierarchy
self.hierarchy.update(hierarchy or {})
def __getattr__(self, item):
return getattr(self.hierarchy[item], item)

123
fuel_agent_ci/fuel_agent_ci/drivers/common_driver.py Normal file
View File

@ -0,0 +1,123 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import requests
from fuel_agent_ci import utils
def artifact_get(artifact):
with open(os.path.join(artifact.env.envdir, artifact.path), 'wb') as f:
for chunk in requests.get(
artifact.url, stream=True).iter_content(1048576):
f.write(chunk)
f.flush()
utils.execute(artifact.unpack, cwd=artifact.env.envdir)
def artifact_clean(artifact):
utils.execute(artifact.clean, cwd=artifact.env.envdir)
def artifact_status(artifact):
return os.path.isfile(os.path.join(artifact.env.envdir, artifact.path))
def dhcp_start(*args, **kwargs):
raise NotImplementedError
def dhcp_stop(*args, **kwargs):
raise NotImplementedError
def dhcp_status(*args, **kwargs):
raise NotImplementedError
def http_start(*args, **kwargs):
raise NotImplementedError
def http_stop(*args, **kwargs):
raise NotImplementedError
def http_status(*args, **kwargs):
raise NotImplementedError
def net_start(*args, **kwargs):
raise NotImplementedError
def net_stop(*args, **kwargs):
raise NotImplementedError
def net_status(*args, **kwargs):
raise NotImplementedError
def repo_clone(*args, **kwargs):
raise NotImplementedError
def repo_clean(*args, **kwargs):
raise NotImplementedError
def repo_status(*args, **kwargs):
raise NotImplementedError
def ssh_status(*args, **kwargs):
raise NotImplementedError
def ssh_put_content(*args, **kwargs):
raise NotImplementedError
def ssh_put_file(*args, **kwargs):
raise NotImplementedError
def ssh_run(*args, **kwargs):
raise NotImplementedError
def tftp_start(*args, **kwargs):
raise NotImplementedError
def tftp_stop(*args, **kwargs):
raise NotImplementedError
def tftp_status(*args, **kwargs):
raise NotImplementedError
def vm_start(*args, **kwargs):
raise NotImplementedError
def vm_stop(*args, **kwargs):
raise NotImplementedError
def vm_status(*args, **kwargs):
raise NotImplementedError

98
fuel_agent_ci/fuel_agent_ci/drivers/fabric_driver.py Normal file
View File

@ -0,0 +1,98 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
import sys
import tempfile
from fabric import api as fab
LOG = logging.getLogger(__name__)
def ssh_status(ssh):
LOG.debug('Trying to get ssh status')
with fab.settings(
host_string=ssh.host,
user=ssh.user,
key_filename=os.path.join(ssh.env.envdir, ssh.key_filename),
timeout=ssh.timeout):
try:
with fab.hide('running', 'stdout', 'stderr'):
fab.run('echo')
LOG.debug('Ssh connection is available')
return True
except SystemExit:
sys.exit()
except Exception:
LOG.debug('Ssh connection is not available')
return False
def ssh_put_content(ssh, file_content, remote_filename):
LOG.debug('Trying to put content into remote file: %s' % remote_filename)
with fab.settings(
host_string=ssh.host,
user=ssh.user,
key_filename=os.path.join(ssh.env.envdir, ssh.key_filename),
timeout=ssh.timeout):
with tempfile.NamedTemporaryFile() as f:
f.write(file_content)
try:
fab.put(f.file, remote_filename)
except SystemExit:
sys.exit()
except Exception:
LOG.error('Error while putting content into '
'remote file: %s' % remote_filename)
raise
def ssh_put_file(ssh, filename, remote_filename):
LOG.debug('Trying to put file on remote host: '
'local=%s remote=%s' % (filename, remote_filename))
with fab.settings(
host_string=ssh.host,
user=ssh.user,
key_filename=os.path.join(ssh.env.envdir, ssh.key_filename),
timeout=ssh.timeout):
try:
fab.put(filename, remote_filename)
except SystemExit:
sys.exit()
except Exception:
LOG.error('Error while putting file on remote host: '
'local=%s remote=%s' % (filename, remote_filename))
raise
def ssh_run(ssh, command, command_timeout=10):
LOG.debug('Trying to run command on remote host: %s' % command)
with fab.settings(
host_string=ssh.host,
user=ssh.user,
key_filename=os.path.join(ssh.env.envdir, ssh.key_filename),
timeout=ssh.timeout,
command_timeout=command_timeout,
warn_only=True):
try:
with fab.hide('running', 'stdout', 'stderr'):
return fab.run(command, pty=True)
except SystemExit:
sys.exit()
except Exception:
LOG.error('Error while putting file on remote host: '
'%s' % command)
raise

208
fuel_agent_ci/fuel_agent_ci/drivers/libvirt_driver.py
View File

@ -343,80 +343,164 @@ class LibvirtDriver(object):
stream.finish()
def env_define(env, drv=None):
def net_start(net, drv=None):
if drv is None:
drv = LibvirtDriver()
LOG.debug('Defining environment: %s' % env.name)
for network in env.networks:
netname = env.name + '_' + network.name
LOG.debug('Defining network: %s' % netname)
network_kwargs = {
'bridge_name': network.bridge,
LOG.debug('Starting network: %s' % net.name)
netname = net.env.name + '_' + net.name
net_kwargs = {
'bridge_name': net.bridge,
'forward_mode': 'nat',
'ip_address': network.ip,
'ip_address': net.ip,
}
if env.tftp and env.tftp.network == network.name:
network_kwargs['tftp_root'] = env.tftp.tftp_root
if env.dhcp and env.dhcp.network == network.name:
network_kwargs['dhcp'] = {
'start': env.dhcp.start,
'end': env.dhcp.end,
tftp = net.env.tftp_by_network(net.name)
if tftp:
net_kwargs['tftp_root'] = os.path.join(
net.env.envdir, tftp.tftp_root)
dhcp = net.env.dhcp_by_network(net.name)
if dhcp:
net_kwargs['dhcp'] = {
'start': dhcp.begin,
'end': dhcp.end,
}
if env.dhcp.bootp:
network_kwargs['dhcp']['bootp'] = env.dhcp.bootp
if env.dhcp.hosts:
network_kwargs['dhcp']['hosts'] = env.dhcp.hosts
drv.net_define(netname, **network_kwargs)
if dhcp.bootp:
net_kwargs['dhcp']['bootp'] = dhcp.bootp
if dhcp.hosts:
net_kwargs['dhcp']['hosts'] = dhcp.hosts
drv.net_define(netname, **net_kwargs)
drv.net_start(drv.net_uuid_by_name(netname))
for vm in env.vms:
vmname = env.name + '_' + vm.name
disks = []
for num, disk in enumerate(vm.disks):
disk_name = vmname + '_%s' % num
order = 'abcdefghijklmnopqrstuvwxyz'
if disk.base:
drv.vol_create(disk_name, base=disk.base)
else:
drv.vol_create(disk_name, capacity=disk.size)
disks.append({
'source_file': drv.vol_path(disk_name),
'target_dev': 'sd%s' % order[num],
'target_bus': 'scsi',
})
interfaces = []
for interface in vm.interfaces:
interfaces.append({
'type': 'network',
'source_network': env.name + '_' + interface.network,
'mac_address': interface.mac
})
drv.define(vmname, boot=vm.boot, disks=disks, interfaces=interfaces)
drv.start(drv.uuid_by_name(vmname))
def env_undefine(env, drv=None):
def net_stop(net, drv=None):
if drv is None:
drv = LibvirtDriver()
for vm in env.vms:
vmname = env.name + '_' + vm.name
if vmname in drv.list():
uuid = drv.uuid_by_name(vmname)
if vmname in drv.list_active():
drv.destroy(uuid)
drv.undefine(uuid)
for volname in [v for v in drv.vol_list() if v.startswith(vmname)]:
drv.vol_delete(volname)
for network in env.networks:
netname = env.name + '_' + network.name
LOG.debug('Stopping net: %s' % net.name)
netname = net.env.name + '_' + net.name
if netname in drv.net_list():
uuid = drv.net_uuid_by_name(netname)
if netname in drv.net_list_active():
drv.net_destroy(uuid)
drv.net_undefine(uuid)
def net_status(net, drv=None):
if drv is None:
drv = LibvirtDriver()
return (net.env.name + '_' + net.name in drv.net_list_active())
def vm_start(vm, drv=None):
if drv is None:
drv = LibvirtDriver()
LOG.debug('Starting vm: %s' % vm.name)
vmname = vm.env.name + '_' + vm.name
if vm.env.name not in drv.pool_list():
LOG.debug('Defining volume pool %s' % vm.env.name)
drv.pool_define(vm.env.name, os.path.join(vm.env.envdir, 'volumepool'))
if vm.env.name not in drv.pool_list_active():
LOG.debug('Starting volume pool %s' % vm.env.name)
drv.pool_start(drv.pool_uuid_by_name(vm.env.name))
disks = []
for num, disk in enumerate(vm.disks):
disk_name = vmname + '_%s' % num
order = 'abcdefghijklmnopqrstuvwxyz'
if disk_name not in drv.vol_list(pool_name=vm.env.name):
if disk.base:
LOG.debug('Creating vm disk: pool=%s vol=%s base=%s' %
(vm.env.name, disk_name, disk.base))
drv.vol_create(disk_name, base=disk.base,
pool_name=vm.env.name)
else:
LOG.debug('Creating empty vm disk: pool=%s vol=%s '
'capacity=%s' % (vm.env.name, disk_name, disk.size))
drv.vol_create(disk_name, capacity=disk.size,
pool_name=vm.env.name)
disks.append({
'source_file': drv.vol_path(disk_name, pool_name=vm.env.name),
'target_dev': 'sd%s' % order[num],
'target_bus': 'scsi',
})
interfaces = []
for interface in vm.interfaces:
LOG.debug('Creating vm interface: net=%s mac=%s' %
(vm.env.name + '_' + interface.network, interface.mac))
interfaces.append({
'type': 'network',
'source_network': vm.env.name + '_' + interface.network,
'mac_address': interface.mac
})
LOG.debug('Defining vm %s' % vm.name)
drv.define(vmname, boot=vm.boot, disks=disks, interfaces=interfaces)
LOG.debug('Starting vm %s' % vm.name)
drv.start(drv.uuid_by_name(vmname))
def vm_stop(vm, drv=None):
if drv is None:
drv = LibvirtDriver()
LOG.debug('Stopping vm: %s' % vm.name)
vmname = vm.env.name + '_' + vm.name
if vmname in drv.list():
uuid = drv.uuid_by_name(vmname)
if vmname in drv.list_active():
LOG.debug('Destroying vm: %s' % vm.name)
drv.destroy(uuid)
LOG.debug('Undefining vm: %s' % vm.name)
drv.undefine(uuid)
for volname in [v for v in drv.vol_list(pool_name=vm.env.name)
if v.startswith(vmname)]:
LOG.debug('Deleting vm disk: pool=%s vol=%s' % (vm.env.name, volname))
drv.vol_delete(volname, pool_name=vm.env.name)
if not drv.vol_list(pool_name=vm.env.name):
LOG.debug('Deleting volume pool: %s' % vm.env.name)
if vm.env.name in drv.pool_list():
uuid = drv.pool_uuid_by_name(vm.env.name)
if vm.env.name in drv.pool_list_active():
LOG.debug('Destroying pool: %s' % vm.env.name)
drv.pool_destroy(uuid)
if vm.env.name in drv.pool_list():
LOG.debug('Undefining pool: %s' % vm.env.name)
drv.pool_undefine(uuid)
def vm_status(vm, drv=None):
if drv is None:
drv = LibvirtDriver()
return (vm.env.name + '_' + vm.name in drv.list_active())
def dhcp_start(dhcp):
"""This feature is implemented in net_start
"""
pass
def dhcp_stop(dhcp):
"""This feature is implemented is net_stop
"""
pass
def dhcp_status(dhcp):
return dhcp.env.net_by_name(dhcp.network).status()
def tftp_start(tftp):
"""This feature is implemented is net_start
"""
pass
def tftp_stop(tftp):
"""This feature is implemented is net_stop
"""
pass
def tftp_status(tftp):
return tftp.env.net_by_name(tftp.network).status()

37
fuel_agent_ci/fuel_agent_ci/drivers/pygit2_driver.py Normal file
View File

@ -0,0 +1,37 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import pygit2
from fuel_agent_ci import utils
def repo_clone(repo):
return pygit2.clone_repository(
repo.url, os.path.join(repo.env.envdir, repo.path),
checkout_branch=repo.branch)
def repo_clean(repo):
utils.execute('rm -rf %s' % os.path.join(repo.env.envdir, repo.path))
def repo_status(repo):
try:
pygit2.discover_repository(os.path.join(repo.env.envdir, repo.path))
except KeyError:
return False
return True

214
fuel_agent_ci/fuel_agent_ci/drivers/simple_http_driver.py Normal file
View File

@ -0,0 +1,214 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import atexit
import BaseHTTPServer
import logging
import multiprocessing
import os
import signal
import SimpleHTTPServer
import sys
import time
import requests
LOG = logging.getLogger(__name__)
class Cwd(object):
def __init__(self, path):
self.path = path
self.orig_path = os.getcwd()
def __enter__(self):
os.chdir(self.path)
def __exit__(self, exc_type, exc_val, exc_tb):
os.chdir(self.orig_path)
class CustomHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def do_GET(self):
if self.path == self.server.parent.shutdown_url:
LOG.info('Shutdown request has been received: %s' % (self.path))
self.send_response(200)
self.end_headers()
self.server.parent.stop_self()
elif self.path == self.server.parent.status_url:
LOG.info('Status request has been received: %s' % (self.path))
self.send_response(200)
self.end_headers()
else:
with Cwd(self.server.parent.rootpath):
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
def do_HEAD(self):
with Cwd(self.server.parent.rootpath):
SimpleHTTPServer.SimpleHTTPRequestHandler.do_HEAD(self)
class CustomHTTPServer(object):
def __init__(self, host, port, rootpath,
shutdown_url='/shutdown',
status_url='/status',
piddir='/var/run',
pidfile='custom_httpd.pid',
stdin=None, stdout=None, stderr=None):
self.host = str(host)
self.port = int(port)
self.rootpath = rootpath
self.shutdown_url = shutdown_url
self.status_url = status_url
self.stdin = stdin
self.stdout = stdout
self.stderr = stderr
self.pidfile = os.path.join(piddir, pidfile)
# We cannot just inherit BaseHTTPServer.HTTPServer because
# it tries to bind socket during initialization but we need it
# to be done during actual launching.
self.server = None
def stop_self(self):
if self.server:
# We cannot use server.shutdown() here because
# it sets _BaseServer__shutdown_request to True
# end wait for _BaseServer__is_shut_down event to be set
# that locks thread forever. We can use shutdown() method
# from outside this thread.
self.server._BaseServer__shutdown_request = True
def daemonize(self):
# in order to avoid http process
# to become zombie we need to fork twice
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError as e:
sys.stderr.write('Error while fork#1 HTTP server: '
'%d (%s)' % (e.errno, e.strerror))
sys.exit(1)
os.chdir('/')
os.setsid()
os.umask(0)
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError as e:
sys.stderr.write('Error while fork#2 HTTP server: '
'%d (%s)' % (e.errno, e.strerror))
sys.exit(1)
if self.stdin:
si = file(self.stdin, 'r')
os.dup2(si.fileno(), sys.stdin.fileno())
if self.stdout:
sys.stdout.flush()
so = file(self.stdout, 'a+')
os.dup2(so.fileno(), sys.stdout.fileno())
if self.stderr:
sys.stderr.flush()
se = file(self.stderr, 'a+', 0)
os.dup2(se.fileno(), sys.stderr.fileno())
atexit.register(self.delpid)
pid = str(os.getpid())
with open(self.pidfile, 'w+') as f:
f.write('%s\n' % pid)
f.flush()
def delpid(self):
os.remove(self.pidfile)
def run(self):
self.server = BaseHTTPServer.HTTPServer(
(self.host, self.port), CustomHTTPRequestHandler)
self.server.parent = self
self.server.serve_forever()
def start(self):
try:
with open(self.pidfile) as f:
pid = int(f.read().strip())
except (IOError, ValueError):
pid = None
if pid:
message = 'pidfile %s already exists. Daemon already running?\n'
sys.stderr.write(message % self.pidfile)
sys.exit(1)
self.daemonize()
self.run()
def stop(self):
try:
with open(self.pidfile) as f:
pid = int(f.read().strip())
except (IOError, ValueError):
pid = None
if not pid:
message = 'pidfile %s does not exist. Daemon not running?\n'
sys.stderr.write(message % self.pidfile)
return
try:
while True:
os.kill(pid, signal.SIGTERM)
time.sleep(1)
except OSError as err:
err = str(err)
if err.find('No such process') > 0:
if os.path.exists(self.pidfile):
os.remove(self.pidfile)
else:
sys.stdout.write(str(err))
sys.exit(1)
def http_start(http):
def start():
server = CustomHTTPServer(
http.env.net_by_name(http.network).ip, http.port,
os.path.join(http.env.envdir, http.http_root),
status_url=http.status_url, shutdown_url=http.shutdown_url,
pidfile=os.path.join(http.env.envdir,
http.env.name + '_custom_httpd.pid'))
server.start()
multiprocessing.Process(target=start).start()
def http_stop(http):
if http_status(http):
requests.get(
'http://%s:%s%s' % (http.env.net_by_name(http.network).ip,
http.port, http.shutdown_url))
def http_status(http):
try:
status = requests.get(
'http://%s:%s%s' % (http.env.net_by_name(http.network).ip,
http.port, http.status_url),
timeout=1
)
if status.status_code == 200:
return True
except Exception:
pass
return False

20
fuel_agent_ci/fuel_agent_ci/manager.py
View File

@ -12,18 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from fuel_agent_ci.drivers import libvirt_driver
from fuel_agent_ci import objects
import logging
from fuel_agent_ci.objects.environment import Environment
LOG = logging.getLogger(__name__)
class Manager(object):
def __init__(self, data):
self.data = data
self.driver = libvirt_driver
self.env = objects.Environment.new(**self.data)
self.env = Environment.new(**data)
def define(self):
self.driver.env_define(self.env)
def do_item(self, item_type, item_action, item_name=None, **kwargs):
return getattr(
self.env, '%s_%s' % (item_type, item_action))(item_name, **kwargs)
def undefine(self):
self.driver.env_undefine(self.env)
def do_env(self, env_action, **kwargs):
return getattr(self.env, env_action)(**kwargs)

28
fuel_agent_ci/fuel_agent_ci/objects/__init__.py
View File

@ -12,15 +12,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from fuel_agent_ci.objects.dhcp import Dhcp
from fuel_agent_ci.objects.environment import Environment
from fuel_agent_ci.objects.http import Http
from fuel_agent_ci.objects.network import Network
from fuel_agent_ci.objects.tftp import Tftp
from fuel_agent_ci.objects.vm import Disk
from fuel_agent_ci.objects.vm import Interface
from fuel_agent_ci.objects.vm import Vm
# This mapping is supposed to be dynamically filled with
# names of objects and their types
OBJECT_TYPES = {}
__all__ = ['Dhcp', 'Environment', 'Http',
'Network', 'Tftp', 'Disk' 'Interface', 'Vm']
class MetaObject(type):
def __init__(self, name, bases, dct):
if '__typename__' in dct:
OBJECT_TYPES[dct['__typename__']] = self
return super(MetaObject, self).__init__(name, bases, dct)
class Object(object):
__metaclass__ = MetaObject
__typename__ = 'object'
@property
def typename(self):
return self.__typename__

46
fuel_agent_ci/fuel_agent_ci/objects/artifact.py Normal file
View File

@ -0,0 +1,46 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from fuel_agent_ci.objects import Object
LOG = logging.getLogger(__name__)
class Artifact(Object):
__typename__ = 'artifact'
def __init__(self, env, name, url, path, unpack=None, clean=None):
self.env = env
self.name = name
self.url = url
self.path = path
self.unpack = unpack
self.clean = clean
def get(self):
if not self.status():
LOG.debug('Getting artifact %s' % self.name)
self.env.driver.artifact_get(self)
def clean(self):
if self.status():
LOG.debug('Cleaning artifact %s' % self.name)
self.env.driver.artifact_clean(self)
def status(self):
status = self.env.driver.artifact_status(self)
LOG.debug('Artifact %s status %s' % (self.name, status))
return status

32
fuel_agent_ci/fuel_agent_ci/objects/dhcp.py
View File

@ -12,9 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
class Dhcp(object):
def __init__(self, start, end, network):
self.start = start
import logging
from fuel_agent_ci.objects import Object
LOG = logging.getLogger(__name__)
class Dhcp(Object):
__typename__ = 'dhcp'
def __init__(self, env, name, begin, end, network):
self.name = name
self.env = env
self.begin = begin
self.end = end
self.network = network
self.hosts = []
@ -28,3 +39,18 @@ class Dhcp(object):
def set_bootp(self, file):
self.bootp = {'file': file}
def start(self):
if not self.status():
LOG.debug('Starting DHCP')
self.env.driver.dhcp_start(self)
def stop(self):
if self.status():
LOG.debug('Stopping DHCP')
self.env.driver.dhcp_stop(self)
def status(self):
status = self.env.driver.dhcp_status(self)
LOG.debug('DHCP status %s' % status)
return status

172
fuel_agent_ci/fuel_agent_ci/objects/environment.py
View File

@ -12,76 +12,164 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
import logging
import os
import tempfile
from fuel_agent_ci.objects.network import Network
from fuel_agent_ci.objects.vm import Vm
from fuel_agent_ci.objects.tftp import Tftp
from fuel_agent_ci import drivers
from fuel_agent_ci.objects.dhcp import Dhcp
from fuel_agent_ci.objects import OBJECT_TYPES
from fuel_agent_ci.objects.vm import Vm
LOG = logging.getLogger(__name__)
class Environment(object):
def __init__(self, name):
def __init__(self, name, envdir, driver=None):
self.name = name
self.networks = []
self.vms = []
self.tftp = None
self.dhcp = None
self.http = None
self.envdir = envdir
self.driver = driver or drivers.Driver()
self.items = []
@classmethod
def new(cls, **kwargs):
LOG.debug('Creating environment: %s' % kwargs['name'])
env = cls(kwargs['name'])
for network_kwargs in kwargs.get('networks', []):
LOG.debug('Creating network: %s' % network_kwargs)
env.add_network(**network_kwargs)
for vm_kwargs in kwargs.get('virtual_machines', []):
LOG.debug('Creating vm: %s' % vm_kwargs)
env.add_vm(**vm_kwargs)
if 'dhcp' in kwargs:
LOG.debug('Creating dhcp server: %s' % kwargs['dhcp'])
env.set_dhcp(**kwargs['dhcp'])
if 'tftp' in kwargs:
LOG.debug('Creating tftp server: %s' % kwargs['tftp'])
env.set_tftp(**kwargs['tftp'])
envdir = kwargs.get('envdir') or os.path.join(
tempfile.gettempdir(), kwargs['name'])
if not os.path.exists(envdir):
LOG.debug('Envdir %s does not exist. Creating envdir.' % envdir)
os.makedirs(envdir)
env = cls(kwargs['name'], envdir)
for item_type in OBJECT_TYPES.keys():
for item_kwargs in kwargs.get(item_type, []):
LOG.debug('Creating %s: %s' % (item_type, item_kwargs))
getattr(env, '%s_add' % item_type)(**item_kwargs)
return env
def add_network(self, **kwargs):
network = Network(**kwargs)
self.networks.append(network)
return network
def __getattr__(self, attr_name):
"""This method maps item_add, item_by_name, item_action attributes into
attributes for particular types like artifact_add or dhcp_by_name.
def add_vm(self, **kwargs):
:param attr_name: Attribute name to map (e.g. net_add, repo_clone)
:returns: Lambda which implements a particular attribute.
"""
try:
item_type, item_action = attr_name.split('_', 1)
except Exception:
raise AttributeError('Attribute %s not found' % attr_name)
else:
if item_action == 'add':
return functools.partial(self.item_add, item_type)
elif item_action == 'by_name':
return functools.partial(self.item_by_name, item_type)
else:
return functools.partial(self.item_action,
item_type, item_action)
def item_add(self, item_type, **kwargs):
if self.item_by_name(item_type, kwargs.get('name')):
raise Exception('Error while adding item: %s %s already exist' %
(item_type, kwargs.get('name')))
item = OBJECT_TYPES[item_type](env=self, **kwargs)
self.items.append(item)
return item
def vm_add(self, **kwargs):
if self.item_by_name('vm', kwargs.get('name')):
raise Exception('Error while adding vm: vm %s already exist' %
kwargs.get('name'))
disks = kwargs.pop('disks', [])
interfaces = kwargs.pop('interfaces', [])
vm = Vm(**kwargs)
vm = Vm(env=self, **kwargs)
for disk_kwargs in disks:
vm.add_disk(**disk_kwargs)
for interface_kwargs in interfaces:
vm.add_interface(**interface_kwargs)
self.vms.append(vm)
self.items.append(vm)
return vm
def set_tftp(self, **kwargs):
if not kwargs['tftp_root'].startswith('/'):
kwargs['tftp_root'] = os.path.abspath(kwargs['tftp_root'])
self.tftp = Tftp(**kwargs)
return self.tftp
def set_dhcp(self, **kwargs):
def dhcp_add(self, **kwargs):
if self.item_by_name('dhcp', kwargs.get('name')):
raise Exception('Error while adding dhcp: dhcp %s already exist' %
kwargs.get('name'))
hosts = kwargs.pop('hosts', [])
bootp_kwargs = kwargs.pop('bootp', None)
self.dhcp = Dhcp(**kwargs)
dhcp = Dhcp(env=self, **kwargs)
for host_kwargs in hosts:
self.dhcp.add_host(**host_kwargs)
dhcp.add_host(**host_kwargs)
if bootp_kwargs is not None:
self.dhcp.set_bootp(**bootp_kwargs)
return self.dhcp
dhcp.set_bootp(**bootp_kwargs)
self.items.append(dhcp)
return dhcp
def set_http(self, **kwargs):
raise NotImplementedError
def item_by_name(self, item_type, item_name):
found = filter(
lambda x: x.typename == item_type and x.name == item_name,
self.items
)
if not found or len(found) > 1:
LOG.debug('Item %s %s not found' % (item_type, item_name))
return None
return found[0]
def item_action(self, item_type, item_action, item_name=None, **kwargs):
if item_name:
item = self.item_by_name(item_type, item_name)
return {item_name: getattr(item, item_action)(**kwargs)}
else:
result = {}
for item in [i for i in self.items if i.typename == item_type]:
LOG.debug('Trying to do action on item: '
'type=%s name=%s action=%s' %
(item_type, item.name, item_action))
result[item.name] = getattr(item, item_action)(**kwargs)
return result
# TODO(kozhukalov): implement this method as classmethod in tftp object
def tftp_by_network(self, network):
found = filter(
lambda x: x.typename == 'tftp' and x.network == network,
self.items
)
if not found or len(found) > 1:
LOG.debug('Tftp not found')
return None
return found[0]
# TODO(kozhukalov): implement this method as classmethod in dhcp object
def dhcp_by_network(self, network):
found = filter(
lambda x: x.typename == 'dhcp' and x.network == network,
self.items
)
if not found or len(found) > 1:
LOG.debug('Dhcp not found')
return None
return found[0]
def start(self):
LOG.debug('Starting environment')
self.artifact_get()
self.repo_clone()
self.net_start()
self.tftp_start()
self.dhcp_start()
self.http_start()
self.vm_start()
def stop(self, artifact_clean=False, repo_clean=False):
LOG.debug('Stopping environment')
self.vm_stop()
self.tftp_stop()
self.dhcp_stop()
self.http_stop()
self.net_stop()
if artifact_clean:
self.artifact_clean()
if repo_clean:
self.repo_clean()
def status(self):
return all((item.status() for item in self.items))

36
fuel_agent_ci/fuel_agent_ci/objects/http.py
View File

@ -12,5 +12,37 @@
# See the License for the specific language governing permissions and
# limitations under the License.
class Http(object):
pass
import logging
from fuel_agent_ci.objects import Object
LOG = logging.getLogger(__name__)
class Http(Object):
__typename__ = 'http'
def __init__(self, env, name, http_root, port, network,
status_url='/status', shutdown_url='/shutdown'):
self.name = name
self.env = env
self.http_root = http_root
self.port = port
self.network = network
self.status_url = status_url
self.shutdown_url = shutdown_url
def start(self):
if not self.status():
LOG.debug('Starting HTTP server')
self.env.driver.http_start(self)
def stop(self):
if self.status():
LOG.debug('Stopping HTTP server')
self.env.driver.http_stop(self)
def status(self):
status = self.env.driver.http_status(self)
LOG.debug('HTTP status %s' % status)
return status

45
fuel_agent_ci/fuel_agent_ci/objects/net.py Normal file
View File

@ -0,0 +1,45 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from fuel_agent_ci.objects import Object
LOG = logging.getLogger(__name__)
class Net(Object):
__typename__ = 'net'
def __init__(self, env, name, bridge, ip, forward):
self.env = env
self.name = name
self.bridge = bridge
self.ip = ip
self.forward = forward
def start(self):
if not self.status():
LOG.debug('Starting network %s' % self.name)
self.env.driver.net_start(self)
def stop(self):
if self.status():
LOG.debug('Stopping network %s' % self.name)
self.env.driver.net_stop(self)
def status(self):
status = self.env.driver.net_status(self)
LOG.debug('Network %s status %s' % (self.name, status))
return status

45
fuel_agent_ci/fuel_agent_ci/objects/repo.py Normal file
View File

@ -0,0 +1,45 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from fuel_agent_ci.objects import Object
LOG = logging.getLogger(__name__)
class Repo(Object):
__typename__ = 'repo'
def __init__(self, env, name, url, path, branch='master'):
self.env = env
self.name = name
self.url = url
self.path = path
self.branch = branch
def clone(self):
if not self.status():
LOG.debug('Cloning repo %s' % self.name)
self.env.driver.repo_clone(self)
def clean(self):
if self.status():
LOG.debug('Cleaning repo %s' % self.name)
self.env.driver.repo_clean(self)
def status(self):
status = self.env.driver.repo_status(self)
LOG.debug('Repo %s status %s' % (self.name, status))
return status

66
fuel_agent_ci/fuel_agent_ci/objects/ssh.py Normal file
View File

@ -0,0 +1,66 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import time
from fuel_agent_ci.objects import Object
LOG = logging.getLogger(__name__)
class Ssh(Object):
__typename__ = 'ssh'
def __init__(self, env, name, host, key_filename, user='root', timeout=5):
self.env = env
self.name = name
self.host = host
self.user = user
self.key_filename = key_filename
self.timeout = timeout
def status(self):
status = self.env.driver.ssh_status(self)
LOG.debug('SSH %s status %s' % (self.name, status))
return status
def put_content(self, content, remote_filename):
if self.status():
LOG.debug('Putting content %s' % self.name)
self.env.driver.ssh_put_content(self, content, remote_filename)
raise Exception('Wrong ssh status: %s' % self.name)
def put_file(self, filename, remote_filename):
if self.status():
LOG.debug('Putting file %s' % self.name)
self.env.driver.ssh_put_file(self, filename, remote_filename)
raise Exception('Wrong ssh status: %s' % self.name)
def run(self, command, command_timeout=10):
if self.status():
LOG.debug('Running command %s' % self.name)
return self.env.driver.ssh_run(self, command, command_timeout)
raise Exception('Wrong ssh status: %s' % self.name)
def wait(self, timeout=200):
begin_time = time.time()
# this loop does not have sleep statement
# because it relies on self.timeout which is by default 5 seconds
while time.time() - begin_time < timeout:
if self.status(self):
return True
LOG.debug('Waiting for ssh connection to be '
'available: %s' % self.name)
return False

30
fuel_agent_ci/fuel_agent_ci/objects/tftp.py
View File

@ -12,7 +12,33 @@
# See the License for the specific language governing permissions and
# limitations under the License.
class Tftp(object):
def __init__(self, tftp_root, network):
import logging
from fuel_agent_ci.objects import Object
LOG = logging.getLogger(__name__)
class Tftp(Object):
__typename__ = 'tftp'
def __init__(self, env, name, tftp_root, network):
self.name = name
self.env = env
self.tftp_root = tftp_root
self.network = network
def start(self):
if not self.status():
LOG.debug('Starting TFTP')
self.env.driver.tftp_start(self)
def stop(self):
if self.status():
LOG.debug('Stopping TFTP')
self.env.driver.tftp_stop(self)
def status(self):
status = self.env.driver.tftp_status(self)
LOG.debug('TFTP status %s' % status)
return status

28
fuel_agent_ci/fuel_agent_ci/objects/vm.py
View File

@ -12,9 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
class Vm(object):
def __init__(self, name, boot=None):
from fuel_agent_ci.objects import Object
LOG = logging.getLogger(__name__)
class Vm(Object):
__typename__ = 'vm'
def __init__(self, env, name, boot=None):
self.env = env
self.name = name
self.interfaces = []
self.disks = []
@ -36,6 +45,21 @@ class Vm(object):
self.disks.append(disk)
return disk
def start(self):
if not self.status():
LOG.debug('Starting virtual machine %s' % self.name)
self.env.driver.vm_start(self)
def stop(self):
if self.status():
LOG.debug('Stopping virtual machine %s' % self.name)
self.env.driver.vm_stop(self)
def status(self):
status = self.env.driver.vm_status(self)
LOG.debug('Virtual machine %s status %s' % (self.name, status))
return status
class Interface(object):
def __init__(self, mac, network):

7
fuel_agent_ci/fuel_agent_ci/objects/network.py → fuel_agent_ci/fuel_agent_ci/tests/__init__.py
View File

@ -11,10 +11,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class Network(object):
def __init__(self, name, bridge, ip, forward):
self.name = name
self.bridge = bridge
self.ip = ip
self.forward = forward

48
fuel_agent_ci/fuel_agent_ci/utils.py
View File

@ -12,14 +12,60 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
from random import choice
import re
import shlex
import string
import subprocess
LOG = logging.getLogger(__name__)
def genmac(start=None):
LOG.debug('Generating mac address')
if start is None:
start = u'00:16:3e:'
chars = string.digits + 'abcdef'
return start + u':'.join([
mac = start + u':'.join([
'{0}{1}'.format(choice(chars), choice(chars)) for _ in xrange(3)])
LOG.debug('Generated mac: %s' % mac)
return mac
def execute(command, to_filename=None, cwd=None):
LOG.debug('Trying to execute command: %s', command)
commands = [c.strip() for c in re.split(ur'\|', command)]
env = os.environ
env['PATH'] = '/bin:/usr/bin:/sbin:/usr/sbin'
to_file = None
if to_filename:
to_file = open(to_filename, 'wb')
process = []
for c in commands:
try:
# NOTE(eli): Python's shlex implementation doesn't like unicode.
# We have to convert to ascii before shlex'ing the command.
# http://bugs.python.org/issue6988
encoded_command = c.encode('ascii')
process.append(subprocess.Popen(
shlex.split(encoded_command),
env=env,
stdin=(process[-1].stdout if process else None),
stdout=(to_file
if (len(process) == len(commands) - 1) and to_file
else subprocess.PIPE),
stderr=(subprocess.PIPE),
cwd=cwd
))
except OSError as e:
return (1, '', '{0}\n'.format(e))
if len(process) >= 2:
process[-2].stdout.close()
stdout, stderr = process[-1].communicate()
return (process[-1].returncode, stdout, stderr)

2
fuel_agent_ci/requirements.txt
View File

@ -3,3 +3,5 @@ ipaddr>=2.1.11
libvirt-python>=1.2.5
xmlbuilder>=1.0
PyYAML>=3.11
# pygit2>=0.20.3
venvgit2>=0.20.3.0

56
fuel_agent_ci/samples/ci_environment.yaml
View File

@ -1,13 +1,53 @@
name: "fuel_agent_ci"
envdir: "/var/tmp/fuel_agent_ci"
networks:
net:
- name: "net"
ip: "10.250.2.1"
bridge: "ci"
forward: "nat"
vm:
- name: "vm"
interfaces:
- mac: "52:54:a5:45:65:ae"
network: "net"
disks:
- size: "10240"
boot: "network"
ssh:
- name: "vm"
host: "10.250.2.20"
user: "root"
key_filename: "ssh/id_rsa"
repo:
- name: "fuel_agent"
url: "https://github.com/stackforge/fuel-web.git"
branch: "master"
path: "fuel_agent"
artifact:
- name: "tftpboot"
url: "http://desktop:9090/tftpboot.tgz"
path: "tftpboot.tgz"
unpack: "tar zxf tftpboot.tgz"
clean: "rm -rf tftpboot tftpboot.tgz"
- name: "image"
url: "http://desktop:9090/image.tgz"
path: "image.tgz"
unpack: "tar zxf image.tgz"
clean: "rm -rf image image.tgz"
- name: "ssh"
url: "http://desktop:9090/ssh.tgz"
path: "ssh.tgz"
unpack: "tar zxf ssh.tgz"
clean: "rm -rf ssh ssh.tgz"
dhcp:
start: "10.250.2.2"
- name: "dhcp"
begin: "10.250.2.2"
end: "10.250.2.254"
hosts:
- mac: "52:54:a5:45:65:ae"
@ -18,14 +58,12 @@ dhcp:
network: "net"
tftp:
- name: "tftp"
tftp_root: "tftpboot"
network: "net"
virtual_machines:
- name: "vm"
interfaces:
- mac: "52:54:a5:45:65:ae"
http:
- name: "http"
port: "8888"
http_root: "image"
network: "net"
disks:
- size: "10240"
boot: "network"

2
fuel_agent_ci/tox.ini
View File

@ -17,7 +17,7 @@ downloadcache = ~/cache/pip
[testenv:pep8]
deps = hacking==0.7
commands =
flake8 {posargs:fuel_agent}
flake8 {posargs:fuel_agent_ci}
[testenv:venv]
commands = {posargs:}