From cce8d62485b2351732dfb3a4549fe04fbb44349b Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Tue, 29 Jul 2014 11:05:24 +0400 Subject: [PATCH] Modified fuel_agent_ci Change-Id: If371eb8d3e1dceae37ad5965c9ea8ca3e336fa94 Implements: blueprint image-based-provisioning --- fuel_agent_ci/fuel_agent_ci/cmd/ci.py | 83 +++++-- .../fuel_agent_ci/drivers/__init__.py | 52 +++++ .../fuel_agent_ci/drivers/common_driver.py | 123 ++++++++++ .../fuel_agent_ci/drivers/fabric_driver.py | 98 ++++++++ .../fuel_agent_ci/drivers/libvirt_driver.py | 218 ++++++++++++------ .../fuel_agent_ci/drivers/pygit2_driver.py | 37 +++ .../drivers/simple_http_driver.py | 214 +++++++++++++++++ fuel_agent_ci/fuel_agent_ci/manager.py | 20 +- .../fuel_agent_ci/objects/__init__.py | 28 ++- .../fuel_agent_ci/objects/artifact.py | 46 ++++ fuel_agent_ci/fuel_agent_ci/objects/dhcp.py | 32 ++- .../fuel_agent_ci/objects/environment.py | 172 ++++++++++---- fuel_agent_ci/fuel_agent_ci/objects/http.py | 36 ++- fuel_agent_ci/fuel_agent_ci/objects/net.py | 45 ++++ fuel_agent_ci/fuel_agent_ci/objects/repo.py | 45 ++++ fuel_agent_ci/fuel_agent_ci/objects/ssh.py | 66 ++++++ fuel_agent_ci/fuel_agent_ci/objects/tftp.py | 30 ++- fuel_agent_ci/fuel_agent_ci/objects/vm.py | 28 ++- .../{objects/network.py => tests/__init__.py} | 7 - fuel_agent_ci/fuel_agent_ci/utils.py | 48 +++- fuel_agent_ci/requirements.txt | 2 + fuel_agent_ci/samples/ci_environment.yaml | 74 ++++-- fuel_agent_ci/tox.ini | 2 +- 23 files changed, 1329 insertions(+), 177 deletions(-) create mode 100644 fuel_agent_ci/fuel_agent_ci/drivers/common_driver.py create mode 100644 fuel_agent_ci/fuel_agent_ci/drivers/fabric_driver.py create mode 100644 fuel_agent_ci/fuel_agent_ci/drivers/pygit2_driver.py create mode 100644 fuel_agent_ci/fuel_agent_ci/drivers/simple_http_driver.py create mode 100644 fuel_agent_ci/fuel_agent_ci/objects/artifact.py create mode 100644 fuel_agent_ci/fuel_agent_ci/objects/net.py create mode 100644 fuel_agent_ci/fuel_agent_ci/objects/repo.py create mode 100644 fuel_agent_ci/fuel_agent_ci/objects/ssh.py rename fuel_agent_ci/fuel_agent_ci/{objects/network.py => tests/__init__.py} (76%) diff --git a/fuel_agent_ci/fuel_agent_ci/cmd/ci.py b/fuel_agent_ci/fuel_agent_ci/cmd/ci.py index 2bf785aa2b..cd4c46e76f 100644 --- a/fuel_agent_ci/fuel_agent_ci/cmd/ci.py +++ b/fuel_agent_ci/fuel_agent_ci/cmd/ci.py @@ -14,6 +14,8 @@ import argparse import logging +import signal +import sys import yaml @@ -24,33 +26,88 @@ logging.basicConfig(level=logging.DEBUG) def parse_args(): parser = argparse.ArgumentParser() + parser.add_argument( + '-f', '--file', dest='env_file', action='store', + type=str, help='Environment data file', required=True + ) + subparsers = parser.add_subparsers(dest='action') - create_parser = subparsers.add_parser('create') - create_parser.add_argument( - '-f', '--file', dest='env_file', action='store', - type=str, help='Environment data file', required=True + env_parser = subparsers.add_parser('env') + env_parser.add_argument( + '-a', '--action', dest='env_action', action='store', + type=str, help='Env action', required=True + ) + env_parser.add_argument( + '-k', '--kwargs', dest='env_kwargs', action='store', + type=str, required=False, + help='Env action kwargs, must be valid json or yaml', + ) + env_parser.add_argument( + '-K', '--kwargs_file', dest='env_kwargs_file', action='store', + type=str, required=False, + help='Env action kwargs file, content must be valid json or yaml', ) - destroy_parser = subparsers.add_parser('destroy') - destroy_parser.add_argument( - '-f', '--file', dest='env_file', action='store', - type=str, help='Environment data file', required=True + item_parser = subparsers.add_parser('item') + item_parser.add_argument( + '-t', '--type', dest='item_type', action='store', + type=str, help='Item type', required=True ) + item_parser.add_argument( + '-a', '--action', dest='item_action', action='store', + type=str, help='Item action', required=True + ) + item_parser.add_argument( + '-n', '--name', dest='item_name', action='store', + type=str, help='Item name', required=False + ) + item_parser.add_argument( + '-k', '--kwargs', dest='item_kwargs', action='store', + type=str, required=False, + help='Item action kwargs, must be valid json or yaml', + ) + item_parser.add_argument( + '-K', '--kwargs_file', dest='item_kwargs_file', action='store', + type=str, required=False, + help='Item action kwargs file, content must be valid json or yaml', + ) + return parser def main(): + def term_handler(signum=None, sigframe=None): + sys.exit() + signal.signal(signal.SIGTERM, term_handler) + signal.signal(signal.SIGINT, term_handler) + parser = parse_args() params, other_params = parser.parse_known_args() - with open(params.env_file, "r") as f: + with open(params.env_file) as f: env_data = yaml.load(f.read()) manager = ci_manager.Manager(env_data) - if params.action == 'create': - manager.define() - elif params.action == 'destroy': - manager.undefine() + # print 'params: %s' % params + # print 'other_params: %s' % other_params + + if params.action == 'env': + kwargs = {} + if params.env_kwargs: + kwargs.update(yaml.load(params.env_kwargs)) + elif params.env_kwargs_file: + with open(params.env_kwargs_file) as f: + kwargs.update(yaml.load(f.read())) + manager.do_env(params.env_action, **kwargs) + elif params.action == 'item': + kwargs = {} + if params.item_kwargs: + kwargs.update(yaml.load(params.item_kwargs)) + elif params.item_kwargs_file: + with open(params.item_kwargs_file) as f: + kwargs.update(yaml.load(f.read())) + manager.do_item(params.item_type, params.item_action, + params.item_name, **kwargs) if __name__ == '__main__': diff --git a/fuel_agent_ci/fuel_agent_ci/drivers/__init__.py b/fuel_agent_ci/fuel_agent_ci/drivers/__init__.py index a2bc2f3ca8..cfd95d92ce 100644 --- a/fuel_agent_ci/fuel_agent_ci/drivers/__init__.py +++ b/fuel_agent_ci/fuel_agent_ci/drivers/__init__.py @@ -11,3 +11,55 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from fuel_agent_ci.drivers import common_driver +from fuel_agent_ci.drivers import fabric_driver +from fuel_agent_ci.drivers import libvirt_driver +from fuel_agent_ci.drivers import pygit2_driver +from fuel_agent_ci.drivers import simple_http_driver + + +class Driver(object): + default_hierarchy = { + # these methods are from common_driver + 'artifact_get': common_driver, + 'artifact_clean': common_driver, + 'artifact_status': common_driver, + + # these methods are from fabric_driver + 'ssh_status': fabric_driver, + 'ssh_put_content': fabric_driver, + 'ssh_put_file': fabric_driver, + 'ssh_run': fabric_driver, + + # these methods are from libvirt_driver + 'net_start': libvirt_driver, + 'net_stop': libvirt_driver, + 'net_status': libvirt_driver, + 'vm_start': libvirt_driver, + 'vm_stop': libvirt_driver, + 'vm_status': libvirt_driver, + 'dhcp_start': libvirt_driver, + 'dhcp_stop': libvirt_driver, + 'dhcp_status': libvirt_driver, + 'tftp_start': libvirt_driver, + 'tftp_stop': libvirt_driver, + 'tftp_status': libvirt_driver, + + # these methods are from pygit2_driver + 'repo_clone': pygit2_driver, + 'repo_clean': pygit2_driver, + 'repo_status': pygit2_driver, + + # these methods are from simple_http_driver + 'http_start': simple_http_driver, + 'http_stop': simple_http_driver, + 'http_status': simple_http_driver, + } + + def __init__(self, hierarchy=None): + self.hierarchy = self.default_hierarchy + self.hierarchy.update(hierarchy or {}) + + def __getattr__(self, item): + return getattr(self.hierarchy[item], item) diff --git a/fuel_agent_ci/fuel_agent_ci/drivers/common_driver.py b/fuel_agent_ci/fuel_agent_ci/drivers/common_driver.py new file mode 100644 index 0000000000..2f8a2678f5 --- /dev/null +++ b/fuel_agent_ci/fuel_agent_ci/drivers/common_driver.py @@ -0,0 +1,123 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import requests + +from fuel_agent_ci import utils + + +def artifact_get(artifact): + with open(os.path.join(artifact.env.envdir, artifact.path), 'wb') as f: + for chunk in requests.get( + artifact.url, stream=True).iter_content(1048576): + f.write(chunk) + f.flush() + utils.execute(artifact.unpack, cwd=artifact.env.envdir) + + +def artifact_clean(artifact): + utils.execute(artifact.clean, cwd=artifact.env.envdir) + + +def artifact_status(artifact): + return os.path.isfile(os.path.join(artifact.env.envdir, artifact.path)) + + +def dhcp_start(*args, **kwargs): + raise NotImplementedError + + +def dhcp_stop(*args, **kwargs): + raise NotImplementedError + + +def dhcp_status(*args, **kwargs): + raise NotImplementedError + + +def http_start(*args, **kwargs): + raise NotImplementedError + + +def http_stop(*args, **kwargs): + raise NotImplementedError + + +def http_status(*args, **kwargs): + raise NotImplementedError + + +def net_start(*args, **kwargs): + raise NotImplementedError + + +def net_stop(*args, **kwargs): + raise NotImplementedError + + +def net_status(*args, **kwargs): + raise NotImplementedError + + +def repo_clone(*args, **kwargs): + raise NotImplementedError + + +def repo_clean(*args, **kwargs): + raise NotImplementedError + + +def repo_status(*args, **kwargs): + raise NotImplementedError + + +def ssh_status(*args, **kwargs): + raise NotImplementedError + + +def ssh_put_content(*args, **kwargs): + raise NotImplementedError + + +def ssh_put_file(*args, **kwargs): + raise NotImplementedError + + +def ssh_run(*args, **kwargs): + raise NotImplementedError + + +def tftp_start(*args, **kwargs): + raise NotImplementedError + + +def tftp_stop(*args, **kwargs): + raise NotImplementedError + + +def tftp_status(*args, **kwargs): + raise NotImplementedError + + +def vm_start(*args, **kwargs): + raise NotImplementedError + + +def vm_stop(*args, **kwargs): + raise NotImplementedError + + +def vm_status(*args, **kwargs): + raise NotImplementedError diff --git a/fuel_agent_ci/fuel_agent_ci/drivers/fabric_driver.py b/fuel_agent_ci/fuel_agent_ci/drivers/fabric_driver.py new file mode 100644 index 0000000000..fc3dc8b486 --- /dev/null +++ b/fuel_agent_ci/fuel_agent_ci/drivers/fabric_driver.py @@ -0,0 +1,98 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import sys +import tempfile + +from fabric import api as fab + +LOG = logging.getLogger(__name__) + + +def ssh_status(ssh): + LOG.debug('Trying to get ssh status') + with fab.settings( + host_string=ssh.host, + user=ssh.user, + key_filename=os.path.join(ssh.env.envdir, ssh.key_filename), + timeout=ssh.timeout): + try: + with fab.hide('running', 'stdout', 'stderr'): + fab.run('echo') + LOG.debug('Ssh connection is available') + return True + except SystemExit: + sys.exit() + except Exception: + LOG.debug('Ssh connection is not available') + return False + + +def ssh_put_content(ssh, file_content, remote_filename): + LOG.debug('Trying to put content into remote file: %s' % remote_filename) + with fab.settings( + host_string=ssh.host, + user=ssh.user, + key_filename=os.path.join(ssh.env.envdir, ssh.key_filename), + timeout=ssh.timeout): + with tempfile.NamedTemporaryFile() as f: + f.write(file_content) + try: + fab.put(f.file, remote_filename) + except SystemExit: + sys.exit() + except Exception: + LOG.error('Error while putting content into ' + 'remote file: %s' % remote_filename) + raise + + +def ssh_put_file(ssh, filename, remote_filename): + LOG.debug('Trying to put file on remote host: ' + 'local=%s remote=%s' % (filename, remote_filename)) + with fab.settings( + host_string=ssh.host, + user=ssh.user, + key_filename=os.path.join(ssh.env.envdir, ssh.key_filename), + timeout=ssh.timeout): + try: + fab.put(filename, remote_filename) + except SystemExit: + sys.exit() + except Exception: + LOG.error('Error while putting file on remote host: ' + 'local=%s remote=%s' % (filename, remote_filename)) + raise + + +def ssh_run(ssh, command, command_timeout=10): + LOG.debug('Trying to run command on remote host: %s' % command) + with fab.settings( + host_string=ssh.host, + user=ssh.user, + key_filename=os.path.join(ssh.env.envdir, ssh.key_filename), + timeout=ssh.timeout, + command_timeout=command_timeout, + warn_only=True): + try: + with fab.hide('running', 'stdout', 'stderr'): + return fab.run(command, pty=True) + except SystemExit: + sys.exit() + except Exception: + LOG.error('Error while putting file on remote host: ' + '%s' % command) + raise diff --git a/fuel_agent_ci/fuel_agent_ci/drivers/libvirt_driver.py b/fuel_agent_ci/fuel_agent_ci/drivers/libvirt_driver.py index 1fd97103c6..12db58a8e7 100644 --- a/fuel_agent_ci/fuel_agent_ci/drivers/libvirt_driver.py +++ b/fuel_agent_ci/fuel_agent_ci/drivers/libvirt_driver.py @@ -343,80 +343,164 @@ class LibvirtDriver(object): stream.finish() -def env_define(env, drv=None): +def net_start(net, drv=None): if drv is None: drv = LibvirtDriver() - - LOG.debug('Defining environment: %s' % env.name) - - for network in env.networks: - netname = env.name + '_' + network.name - LOG.debug('Defining network: %s' % netname) - network_kwargs = { - 'bridge_name': network.bridge, - 'forward_mode': 'nat', - 'ip_address': network.ip, + LOG.debug('Starting network: %s' % net.name) + netname = net.env.name + '_' + net.name + net_kwargs = { + 'bridge_name': net.bridge, + 'forward_mode': 'nat', + 'ip_address': net.ip, + } + tftp = net.env.tftp_by_network(net.name) + if tftp: + net_kwargs['tftp_root'] = os.path.join( + net.env.envdir, tftp.tftp_root) + dhcp = net.env.dhcp_by_network(net.name) + if dhcp: + net_kwargs['dhcp'] = { + 'start': dhcp.begin, + 'end': dhcp.end, } - if env.tftp and env.tftp.network == network.name: - network_kwargs['tftp_root'] = env.tftp.tftp_root - if env.dhcp and env.dhcp.network == network.name: - network_kwargs['dhcp'] = { - 'start': env.dhcp.start, - 'end': env.dhcp.end, - } - if env.dhcp.bootp: - network_kwargs['dhcp']['bootp'] = env.dhcp.bootp - if env.dhcp.hosts: - network_kwargs['dhcp']['hosts'] = env.dhcp.hosts - drv.net_define(netname, **network_kwargs) - drv.net_start(drv.net_uuid_by_name(netname)) + if dhcp.bootp: + net_kwargs['dhcp']['bootp'] = dhcp.bootp + if dhcp.hosts: + net_kwargs['dhcp']['hosts'] = dhcp.hosts + drv.net_define(netname, **net_kwargs) + drv.net_start(drv.net_uuid_by_name(netname)) - for vm in env.vms: - vmname = env.name + '_' + vm.name - disks = [] - for num, disk in enumerate(vm.disks): - disk_name = vmname + '_%s' % num - order = 'abcdefghijklmnopqrstuvwxyz' - if disk.base: - drv.vol_create(disk_name, base=disk.base) - else: - drv.vol_create(disk_name, capacity=disk.size) - disks.append({ - 'source_file': drv.vol_path(disk_name), - 'target_dev': 'sd%s' % order[num], - 'target_bus': 'scsi', - }) - interfaces = [] - for interface in vm.interfaces: - interfaces.append({ - 'type': 'network', - 'source_network': env.name + '_' + interface.network, - 'mac_address': interface.mac - }) - drv.define(vmname, boot=vm.boot, disks=disks, interfaces=interfaces) - drv.start(drv.uuid_by_name(vmname)) - - -def env_undefine(env, drv=None): +def net_stop(net, drv=None): if drv is None: drv = LibvirtDriver() + LOG.debug('Stopping net: %s' % net.name) + netname = net.env.name + '_' + net.name + if netname in drv.net_list(): + uuid = drv.net_uuid_by_name(netname) + if netname in drv.net_list_active(): + drv.net_destroy(uuid) + drv.net_undefine(uuid) - for vm in env.vms: - vmname = env.name + '_' + vm.name - if vmname in drv.list(): - uuid = drv.uuid_by_name(vmname) - if vmname in drv.list_active(): - drv.destroy(uuid) - drv.undefine(uuid) - for volname in [v for v in drv.vol_list() if v.startswith(vmname)]: - drv.vol_delete(volname) +def net_status(net, drv=None): + if drv is None: + drv = LibvirtDriver() + return (net.env.name + '_' + net.name in drv.net_list_active()) - for network in env.networks: - netname = env.name + '_' + network.name - if netname in drv.net_list(): - uuid = drv.net_uuid_by_name(netname) - if netname in drv.net_list_active(): - drv.net_destroy(uuid) - drv.net_undefine(uuid) + +def vm_start(vm, drv=None): + if drv is None: + drv = LibvirtDriver() + LOG.debug('Starting vm: %s' % vm.name) + vmname = vm.env.name + '_' + vm.name + + if vm.env.name not in drv.pool_list(): + LOG.debug('Defining volume pool %s' % vm.env.name) + drv.pool_define(vm.env.name, os.path.join(vm.env.envdir, 'volumepool')) + if vm.env.name not in drv.pool_list_active(): + LOG.debug('Starting volume pool %s' % vm.env.name) + drv.pool_start(drv.pool_uuid_by_name(vm.env.name)) + + disks = [] + for num, disk in enumerate(vm.disks): + disk_name = vmname + '_%s' % num + order = 'abcdefghijklmnopqrstuvwxyz' + if disk_name not in drv.vol_list(pool_name=vm.env.name): + if disk.base: + LOG.debug('Creating vm disk: pool=%s vol=%s base=%s' % + (vm.env.name, disk_name, disk.base)) + drv.vol_create(disk_name, base=disk.base, + pool_name=vm.env.name) + else: + LOG.debug('Creating empty vm disk: pool=%s vol=%s ' + 'capacity=%s' % (vm.env.name, disk_name, disk.size)) + drv.vol_create(disk_name, capacity=disk.size, + pool_name=vm.env.name) + disks.append({ + 'source_file': drv.vol_path(disk_name, pool_name=vm.env.name), + 'target_dev': 'sd%s' % order[num], + 'target_bus': 'scsi', + }) + + interfaces = [] + for interface in vm.interfaces: + LOG.debug('Creating vm interface: net=%s mac=%s' % + (vm.env.name + '_' + interface.network, interface.mac)) + interfaces.append({ + 'type': 'network', + 'source_network': vm.env.name + '_' + interface.network, + 'mac_address': interface.mac + }) + LOG.debug('Defining vm %s' % vm.name) + drv.define(vmname, boot=vm.boot, disks=disks, interfaces=interfaces) + LOG.debug('Starting vm %s' % vm.name) + drv.start(drv.uuid_by_name(vmname)) + + +def vm_stop(vm, drv=None): + if drv is None: + drv = LibvirtDriver() + LOG.debug('Stopping vm: %s' % vm.name) + vmname = vm.env.name + '_' + vm.name + if vmname in drv.list(): + uuid = drv.uuid_by_name(vmname) + if vmname in drv.list_active(): + LOG.debug('Destroying vm: %s' % vm.name) + drv.destroy(uuid) + LOG.debug('Undefining vm: %s' % vm.name) + drv.undefine(uuid) + + for volname in [v for v in drv.vol_list(pool_name=vm.env.name) + if v.startswith(vmname)]: + LOG.debug('Deleting vm disk: pool=%s vol=%s' % (vm.env.name, volname)) + drv.vol_delete(volname, pool_name=vm.env.name) + + if not drv.vol_list(pool_name=vm.env.name): + LOG.debug('Deleting volume pool: %s' % vm.env.name) + if vm.env.name in drv.pool_list(): + uuid = drv.pool_uuid_by_name(vm.env.name) + if vm.env.name in drv.pool_list_active(): + LOG.debug('Destroying pool: %s' % vm.env.name) + drv.pool_destroy(uuid) + if vm.env.name in drv.pool_list(): + LOG.debug('Undefining pool: %s' % vm.env.name) + drv.pool_undefine(uuid) + + +def vm_status(vm, drv=None): + if drv is None: + drv = LibvirtDriver() + return (vm.env.name + '_' + vm.name in drv.list_active()) + + +def dhcp_start(dhcp): + """This feature is implemented in net_start + """ + pass + + +def dhcp_stop(dhcp): + """This feature is implemented is net_stop + """ + pass + + +def dhcp_status(dhcp): + return dhcp.env.net_by_name(dhcp.network).status() + + +def tftp_start(tftp): + """This feature is implemented is net_start + """ + pass + + +def tftp_stop(tftp): + """This feature is implemented is net_stop + """ + pass + + +def tftp_status(tftp): + return tftp.env.net_by_name(tftp.network).status() diff --git a/fuel_agent_ci/fuel_agent_ci/drivers/pygit2_driver.py b/fuel_agent_ci/fuel_agent_ci/drivers/pygit2_driver.py new file mode 100644 index 0000000000..d283e71ac9 --- /dev/null +++ b/fuel_agent_ci/fuel_agent_ci/drivers/pygit2_driver.py @@ -0,0 +1,37 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pygit2 + +from fuel_agent_ci import utils + + +def repo_clone(repo): + return pygit2.clone_repository( + repo.url, os.path.join(repo.env.envdir, repo.path), + checkout_branch=repo.branch) + + +def repo_clean(repo): + utils.execute('rm -rf %s' % os.path.join(repo.env.envdir, repo.path)) + + +def repo_status(repo): + try: + pygit2.discover_repository(os.path.join(repo.env.envdir, repo.path)) + except KeyError: + return False + return True diff --git a/fuel_agent_ci/fuel_agent_ci/drivers/simple_http_driver.py b/fuel_agent_ci/fuel_agent_ci/drivers/simple_http_driver.py new file mode 100644 index 0000000000..cfd3f82754 --- /dev/null +++ b/fuel_agent_ci/fuel_agent_ci/drivers/simple_http_driver.py @@ -0,0 +1,214 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit +import BaseHTTPServer +import logging +import multiprocessing +import os +import signal +import SimpleHTTPServer +import sys +import time + +import requests + +LOG = logging.getLogger(__name__) + + +class Cwd(object): + def __init__(self, path): + self.path = path + self.orig_path = os.getcwd() + + def __enter__(self): + os.chdir(self.path) + + def __exit__(self, exc_type, exc_val, exc_tb): + os.chdir(self.orig_path) + + +class CustomHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + def do_GET(self): + if self.path == self.server.parent.shutdown_url: + LOG.info('Shutdown request has been received: %s' % (self.path)) + self.send_response(200) + self.end_headers() + self.server.parent.stop_self() + elif self.path == self.server.parent.status_url: + LOG.info('Status request has been received: %s' % (self.path)) + self.send_response(200) + self.end_headers() + else: + with Cwd(self.server.parent.rootpath): + SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) + + def do_HEAD(self): + with Cwd(self.server.parent.rootpath): + SimpleHTTPServer.SimpleHTTPRequestHandler.do_HEAD(self) + + +class CustomHTTPServer(object): + def __init__(self, host, port, rootpath, + shutdown_url='/shutdown', + status_url='/status', + piddir='/var/run', + pidfile='custom_httpd.pid', + stdin=None, stdout=None, stderr=None): + + self.host = str(host) + self.port = int(port) + + self.rootpath = rootpath + self.shutdown_url = shutdown_url + self.status_url = status_url + self.stdin = stdin + self.stdout = stdout + self.stderr = stderr + self.pidfile = os.path.join(piddir, pidfile) + + # We cannot just inherit BaseHTTPServer.HTTPServer because + # it tries to bind socket during initialization but we need it + # to be done during actual launching. + self.server = None + + def stop_self(self): + if self.server: + # We cannot use server.shutdown() here because + # it sets _BaseServer__shutdown_request to True + # end wait for _BaseServer__is_shut_down event to be set + # that locks thread forever. We can use shutdown() method + # from outside this thread. + self.server._BaseServer__shutdown_request = True + + def daemonize(self): + # in order to avoid http process + # to become zombie we need to fork twice + try: + pid = os.fork() + if pid > 0: + sys.exit(0) + except OSError as e: + sys.stderr.write('Error while fork#1 HTTP server: ' + '%d (%s)' % (e.errno, e.strerror)) + sys.exit(1) + + os.chdir('/') + os.setsid() + os.umask(0) + + try: + pid = os.fork() + if pid > 0: + sys.exit(0) + except OSError as e: + sys.stderr.write('Error while fork#2 HTTP server: ' + '%d (%s)' % (e.errno, e.strerror)) + sys.exit(1) + + if self.stdin: + si = file(self.stdin, 'r') + os.dup2(si.fileno(), sys.stdin.fileno()) + if self.stdout: + sys.stdout.flush() + so = file(self.stdout, 'a+') + os.dup2(so.fileno(), sys.stdout.fileno()) + if self.stderr: + sys.stderr.flush() + se = file(self.stderr, 'a+', 0) + os.dup2(se.fileno(), sys.stderr.fileno()) + + atexit.register(self.delpid) + pid = str(os.getpid()) + with open(self.pidfile, 'w+') as f: + f.write('%s\n' % pid) + f.flush() + + def delpid(self): + os.remove(self.pidfile) + + def run(self): + self.server = BaseHTTPServer.HTTPServer( + (self.host, self.port), CustomHTTPRequestHandler) + self.server.parent = self + self.server.serve_forever() + + def start(self): + try: + with open(self.pidfile) as f: + pid = int(f.read().strip()) + except (IOError, ValueError): + pid = None + if pid: + message = 'pidfile %s already exists. Daemon already running?\n' + sys.stderr.write(message % self.pidfile) + sys.exit(1) + self.daemonize() + self.run() + + def stop(self): + try: + with open(self.pidfile) as f: + pid = int(f.read().strip()) + except (IOError, ValueError): + pid = None + if not pid: + message = 'pidfile %s does not exist. Daemon not running?\n' + sys.stderr.write(message % self.pidfile) + return + try: + while True: + os.kill(pid, signal.SIGTERM) + time.sleep(1) + except OSError as err: + err = str(err) + if err.find('No such process') > 0: + if os.path.exists(self.pidfile): + os.remove(self.pidfile) + else: + sys.stdout.write(str(err)) + sys.exit(1) + + +def http_start(http): + def start(): + server = CustomHTTPServer( + http.env.net_by_name(http.network).ip, http.port, + os.path.join(http.env.envdir, http.http_root), + status_url=http.status_url, shutdown_url=http.shutdown_url, + pidfile=os.path.join(http.env.envdir, + http.env.name + '_custom_httpd.pid')) + server.start() + multiprocessing.Process(target=start).start() + + +def http_stop(http): + if http_status(http): + requests.get( + 'http://%s:%s%s' % (http.env.net_by_name(http.network).ip, + http.port, http.shutdown_url)) + + +def http_status(http): + try: + status = requests.get( + 'http://%s:%s%s' % (http.env.net_by_name(http.network).ip, + http.port, http.status_url), + timeout=1 + ) + if status.status_code == 200: + return True + except Exception: + pass + return False diff --git a/fuel_agent_ci/fuel_agent_ci/manager.py b/fuel_agent_ci/fuel_agent_ci/manager.py index 752c6693c4..94fc8de9b1 100644 --- a/fuel_agent_ci/fuel_agent_ci/manager.py +++ b/fuel_agent_ci/fuel_agent_ci/manager.py @@ -12,18 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from fuel_agent_ci.drivers import libvirt_driver -from fuel_agent_ci import objects +import logging + +from fuel_agent_ci.objects.environment import Environment + +LOG = logging.getLogger(__name__) class Manager(object): def __init__(self, data): - self.data = data - self.driver = libvirt_driver - self.env = objects.Environment.new(**self.data) + self.env = Environment.new(**data) - def define(self): - self.driver.env_define(self.env) + def do_item(self, item_type, item_action, item_name=None, **kwargs): + return getattr( + self.env, '%s_%s' % (item_type, item_action))(item_name, **kwargs) - def undefine(self): - self.driver.env_undefine(self.env) + def do_env(self, env_action, **kwargs): + return getattr(self.env, env_action)(**kwargs) diff --git a/fuel_agent_ci/fuel_agent_ci/objects/__init__.py b/fuel_agent_ci/fuel_agent_ci/objects/__init__.py index e93d165c6e..2d15b3691c 100644 --- a/fuel_agent_ci/fuel_agent_ci/objects/__init__.py +++ b/fuel_agent_ci/fuel_agent_ci/objects/__init__.py @@ -12,15 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -from fuel_agent_ci.objects.dhcp import Dhcp -from fuel_agent_ci.objects.environment import Environment -from fuel_agent_ci.objects.http import Http -from fuel_agent_ci.objects.network import Network -from fuel_agent_ci.objects.tftp import Tftp -from fuel_agent_ci.objects.vm import Disk -from fuel_agent_ci.objects.vm import Interface -from fuel_agent_ci.objects.vm import Vm +# This mapping is supposed to be dynamically filled with +# names of objects and their types + +OBJECT_TYPES = {} -__all__ = ['Dhcp', 'Environment', 'Http', - 'Network', 'Tftp', 'Disk' 'Interface', 'Vm'] +class MetaObject(type): + def __init__(self, name, bases, dct): + if '__typename__' in dct: + OBJECT_TYPES[dct['__typename__']] = self + return super(MetaObject, self).__init__(name, bases, dct) + + +class Object(object): + __metaclass__ = MetaObject + __typename__ = 'object' + + @property + def typename(self): + return self.__typename__ diff --git a/fuel_agent_ci/fuel_agent_ci/objects/artifact.py b/fuel_agent_ci/fuel_agent_ci/objects/artifact.py new file mode 100644 index 0000000000..1b60708c2a --- /dev/null +++ b/fuel_agent_ci/fuel_agent_ci/objects/artifact.py @@ -0,0 +1,46 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from fuel_agent_ci.objects import Object + +LOG = logging.getLogger(__name__) + + +class Artifact(Object): + __typename__ = 'artifact' + + def __init__(self, env, name, url, path, unpack=None, clean=None): + self.env = env + self.name = name + self.url = url + self.path = path + self.unpack = unpack + self.clean = clean + + def get(self): + if not self.status(): + LOG.debug('Getting artifact %s' % self.name) + self.env.driver.artifact_get(self) + + def clean(self): + if self.status(): + LOG.debug('Cleaning artifact %s' % self.name) + self.env.driver.artifact_clean(self) + + def status(self): + status = self.env.driver.artifact_status(self) + LOG.debug('Artifact %s status %s' % (self.name, status)) + return status diff --git a/fuel_agent_ci/fuel_agent_ci/objects/dhcp.py b/fuel_agent_ci/fuel_agent_ci/objects/dhcp.py index e8d28712e7..4b2b6456b5 100644 --- a/fuel_agent_ci/fuel_agent_ci/objects/dhcp.py +++ b/fuel_agent_ci/fuel_agent_ci/objects/dhcp.py @@ -12,9 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -class Dhcp(object): - def __init__(self, start, end, network): - self.start = start +import logging + +from fuel_agent_ci.objects import Object + +LOG = logging.getLogger(__name__) + + +class Dhcp(Object): + __typename__ = 'dhcp' + + def __init__(self, env, name, begin, end, network): + self.name = name + self.env = env + self.begin = begin self.end = end self.network = network self.hosts = [] @@ -28,3 +39,18 @@ class Dhcp(object): def set_bootp(self, file): self.bootp = {'file': file} + + def start(self): + if not self.status(): + LOG.debug('Starting DHCP') + self.env.driver.dhcp_start(self) + + def stop(self): + if self.status(): + LOG.debug('Stopping DHCP') + self.env.driver.dhcp_stop(self) + + def status(self): + status = self.env.driver.dhcp_status(self) + LOG.debug('DHCP status %s' % status) + return status diff --git a/fuel_agent_ci/fuel_agent_ci/objects/environment.py b/fuel_agent_ci/fuel_agent_ci/objects/environment.py index 3ef3b072cf..2a513e382e 100644 --- a/fuel_agent_ci/fuel_agent_ci/objects/environment.py +++ b/fuel_agent_ci/fuel_agent_ci/objects/environment.py @@ -12,76 +12,164 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools import logging import os +import tempfile -from fuel_agent_ci.objects.network import Network -from fuel_agent_ci.objects.vm import Vm -from fuel_agent_ci.objects.tftp import Tftp +from fuel_agent_ci import drivers from fuel_agent_ci.objects.dhcp import Dhcp +from fuel_agent_ci.objects import OBJECT_TYPES +from fuel_agent_ci.objects.vm import Vm LOG = logging.getLogger(__name__) class Environment(object): - def __init__(self, name): + def __init__(self, name, envdir, driver=None): self.name = name - self.networks = [] - self.vms = [] - self.tftp = None - self.dhcp = None - self.http = None + self.envdir = envdir + self.driver = driver or drivers.Driver() + self.items = [] @classmethod def new(cls, **kwargs): LOG.debug('Creating environment: %s' % kwargs['name']) - env = cls(kwargs['name']) - for network_kwargs in kwargs.get('networks', []): - LOG.debug('Creating network: %s' % network_kwargs) - env.add_network(**network_kwargs) - for vm_kwargs in kwargs.get('virtual_machines', []): - LOG.debug('Creating vm: %s' % vm_kwargs) - env.add_vm(**vm_kwargs) - if 'dhcp' in kwargs: - LOG.debug('Creating dhcp server: %s' % kwargs['dhcp']) - env.set_dhcp(**kwargs['dhcp']) - if 'tftp' in kwargs: - LOG.debug('Creating tftp server: %s' % kwargs['tftp']) - env.set_tftp(**kwargs['tftp']) + envdir = kwargs.get('envdir') or os.path.join( + tempfile.gettempdir(), kwargs['name']) + if not os.path.exists(envdir): + LOG.debug('Envdir %s does not exist. Creating envdir.' % envdir) + os.makedirs(envdir) + env = cls(kwargs['name'], envdir) + for item_type in OBJECT_TYPES.keys(): + for item_kwargs in kwargs.get(item_type, []): + LOG.debug('Creating %s: %s' % (item_type, item_kwargs)) + getattr(env, '%s_add' % item_type)(**item_kwargs) return env - def add_network(self, **kwargs): - network = Network(**kwargs) - self.networks.append(network) - return network + def __getattr__(self, attr_name): + """This method maps item_add, item_by_name, item_action attributes into + attributes for particular types like artifact_add or dhcp_by_name. - def add_vm(self, **kwargs): + :param attr_name: Attribute name to map (e.g. net_add, repo_clone) + + :returns: Lambda which implements a particular attribute. + """ + try: + item_type, item_action = attr_name.split('_', 1) + except Exception: + raise AttributeError('Attribute %s not found' % attr_name) + else: + if item_action == 'add': + return functools.partial(self.item_add, item_type) + elif item_action == 'by_name': + return functools.partial(self.item_by_name, item_type) + else: + return functools.partial(self.item_action, + item_type, item_action) + + def item_add(self, item_type, **kwargs): + if self.item_by_name(item_type, kwargs.get('name')): + raise Exception('Error while adding item: %s %s already exist' % + (item_type, kwargs.get('name'))) + item = OBJECT_TYPES[item_type](env=self, **kwargs) + self.items.append(item) + return item + + def vm_add(self, **kwargs): + if self.item_by_name('vm', kwargs.get('name')): + raise Exception('Error while adding vm: vm %s already exist' % + kwargs.get('name')) disks = kwargs.pop('disks', []) interfaces = kwargs.pop('interfaces', []) - vm = Vm(**kwargs) + vm = Vm(env=self, **kwargs) for disk_kwargs in disks: vm.add_disk(**disk_kwargs) for interface_kwargs in interfaces: vm.add_interface(**interface_kwargs) - self.vms.append(vm) + self.items.append(vm) return vm - def set_tftp(self, **kwargs): - if not kwargs['tftp_root'].startswith('/'): - kwargs['tftp_root'] = os.path.abspath(kwargs['tftp_root']) - self.tftp = Tftp(**kwargs) - return self.tftp - - def set_dhcp(self, **kwargs): + def dhcp_add(self, **kwargs): + if self.item_by_name('dhcp', kwargs.get('name')): + raise Exception('Error while adding dhcp: dhcp %s already exist' % + kwargs.get('name')) hosts = kwargs.pop('hosts', []) bootp_kwargs = kwargs.pop('bootp', None) - self.dhcp = Dhcp(**kwargs) + dhcp = Dhcp(env=self, **kwargs) for host_kwargs in hosts: - self.dhcp.add_host(**host_kwargs) + dhcp.add_host(**host_kwargs) if bootp_kwargs is not None: - self.dhcp.set_bootp(**bootp_kwargs) - return self.dhcp + dhcp.set_bootp(**bootp_kwargs) + self.items.append(dhcp) + return dhcp - def set_http(self, **kwargs): - raise NotImplementedError + def item_by_name(self, item_type, item_name): + found = filter( + lambda x: x.typename == item_type and x.name == item_name, + self.items + ) + if not found or len(found) > 1: + LOG.debug('Item %s %s not found' % (item_type, item_name)) + return None + return found[0] + def item_action(self, item_type, item_action, item_name=None, **kwargs): + if item_name: + item = self.item_by_name(item_type, item_name) + return {item_name: getattr(item, item_action)(**kwargs)} + else: + result = {} + for item in [i for i in self.items if i.typename == item_type]: + LOG.debug('Trying to do action on item: ' + 'type=%s name=%s action=%s' % + (item_type, item.name, item_action)) + result[item.name] = getattr(item, item_action)(**kwargs) + return result + + # TODO(kozhukalov): implement this method as classmethod in tftp object + def tftp_by_network(self, network): + found = filter( + lambda x: x.typename == 'tftp' and x.network == network, + self.items + ) + if not found or len(found) > 1: + LOG.debug('Tftp not found') + return None + return found[0] + + # TODO(kozhukalov): implement this method as classmethod in dhcp object + def dhcp_by_network(self, network): + found = filter( + lambda x: x.typename == 'dhcp' and x.network == network, + self.items + ) + if not found or len(found) > 1: + LOG.debug('Dhcp not found') + return None + return found[0] + + def start(self): + LOG.debug('Starting environment') + self.artifact_get() + self.repo_clone() + self.net_start() + self.tftp_start() + self.dhcp_start() + self.http_start() + self.vm_start() + + def stop(self, artifact_clean=False, repo_clean=False): + LOG.debug('Stopping environment') + self.vm_stop() + self.tftp_stop() + self.dhcp_stop() + self.http_stop() + self.net_stop() + if artifact_clean: + self.artifact_clean() + if repo_clean: + self.repo_clean() + + def status(self): + return all((item.status() for item in self.items)) diff --git a/fuel_agent_ci/fuel_agent_ci/objects/http.py b/fuel_agent_ci/fuel_agent_ci/objects/http.py index 993a5ae2e6..bf3567dcb7 100644 --- a/fuel_agent_ci/fuel_agent_ci/objects/http.py +++ b/fuel_agent_ci/fuel_agent_ci/objects/http.py @@ -12,5 +12,37 @@ # See the License for the specific language governing permissions and # limitations under the License. -class Http(object): - pass \ No newline at end of file +import logging + +from fuel_agent_ci.objects import Object + +LOG = logging.getLogger(__name__) + + +class Http(Object): + __typename__ = 'http' + + def __init__(self, env, name, http_root, port, network, + status_url='/status', shutdown_url='/shutdown'): + self.name = name + self.env = env + self.http_root = http_root + self.port = port + self.network = network + self.status_url = status_url + self.shutdown_url = shutdown_url + + def start(self): + if not self.status(): + LOG.debug('Starting HTTP server') + self.env.driver.http_start(self) + + def stop(self): + if self.status(): + LOG.debug('Stopping HTTP server') + self.env.driver.http_stop(self) + + def status(self): + status = self.env.driver.http_status(self) + LOG.debug('HTTP status %s' % status) + return status diff --git a/fuel_agent_ci/fuel_agent_ci/objects/net.py b/fuel_agent_ci/fuel_agent_ci/objects/net.py new file mode 100644 index 0000000000..2c7abe8aeb --- /dev/null +++ b/fuel_agent_ci/fuel_agent_ci/objects/net.py @@ -0,0 +1,45 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from fuel_agent_ci.objects import Object + +LOG = logging.getLogger(__name__) + + +class Net(Object): + __typename__ = 'net' + + def __init__(self, env, name, bridge, ip, forward): + self.env = env + self.name = name + self.bridge = bridge + self.ip = ip + self.forward = forward + + def start(self): + if not self.status(): + LOG.debug('Starting network %s' % self.name) + self.env.driver.net_start(self) + + def stop(self): + if self.status(): + LOG.debug('Stopping network %s' % self.name) + self.env.driver.net_stop(self) + + def status(self): + status = self.env.driver.net_status(self) + LOG.debug('Network %s status %s' % (self.name, status)) + return status diff --git a/fuel_agent_ci/fuel_agent_ci/objects/repo.py b/fuel_agent_ci/fuel_agent_ci/objects/repo.py new file mode 100644 index 0000000000..5841666005 --- /dev/null +++ b/fuel_agent_ci/fuel_agent_ci/objects/repo.py @@ -0,0 +1,45 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from fuel_agent_ci.objects import Object + +LOG = logging.getLogger(__name__) + + +class Repo(Object): + __typename__ = 'repo' + + def __init__(self, env, name, url, path, branch='master'): + self.env = env + self.name = name + self.url = url + self.path = path + self.branch = branch + + def clone(self): + if not self.status(): + LOG.debug('Cloning repo %s' % self.name) + self.env.driver.repo_clone(self) + + def clean(self): + if self.status(): + LOG.debug('Cleaning repo %s' % self.name) + self.env.driver.repo_clean(self) + + def status(self): + status = self.env.driver.repo_status(self) + LOG.debug('Repo %s status %s' % (self.name, status)) + return status diff --git a/fuel_agent_ci/fuel_agent_ci/objects/ssh.py b/fuel_agent_ci/fuel_agent_ci/objects/ssh.py new file mode 100644 index 0000000000..0e08d512de --- /dev/null +++ b/fuel_agent_ci/fuel_agent_ci/objects/ssh.py @@ -0,0 +1,66 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import time + +from fuel_agent_ci.objects import Object + +LOG = logging.getLogger(__name__) + + +class Ssh(Object): + __typename__ = 'ssh' + + def __init__(self, env, name, host, key_filename, user='root', timeout=5): + self.env = env + self.name = name + self.host = host + self.user = user + self.key_filename = key_filename + self.timeout = timeout + + def status(self): + status = self.env.driver.ssh_status(self) + LOG.debug('SSH %s status %s' % (self.name, status)) + return status + + def put_content(self, content, remote_filename): + if self.status(): + LOG.debug('Putting content %s' % self.name) + self.env.driver.ssh_put_content(self, content, remote_filename) + raise Exception('Wrong ssh status: %s' % self.name) + + def put_file(self, filename, remote_filename): + if self.status(): + LOG.debug('Putting file %s' % self.name) + self.env.driver.ssh_put_file(self, filename, remote_filename) + raise Exception('Wrong ssh status: %s' % self.name) + + def run(self, command, command_timeout=10): + if self.status(): + LOG.debug('Running command %s' % self.name) + return self.env.driver.ssh_run(self, command, command_timeout) + raise Exception('Wrong ssh status: %s' % self.name) + + def wait(self, timeout=200): + begin_time = time.time() + # this loop does not have sleep statement + # because it relies on self.timeout which is by default 5 seconds + while time.time() - begin_time < timeout: + if self.status(self): + return True + LOG.debug('Waiting for ssh connection to be ' + 'available: %s' % self.name) + return False diff --git a/fuel_agent_ci/fuel_agent_ci/objects/tftp.py b/fuel_agent_ci/fuel_agent_ci/objects/tftp.py index b113e838ec..e025487eb2 100644 --- a/fuel_agent_ci/fuel_agent_ci/objects/tftp.py +++ b/fuel_agent_ci/fuel_agent_ci/objects/tftp.py @@ -12,7 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -class Tftp(object): - def __init__(self, tftp_root, network): +import logging + +from fuel_agent_ci.objects import Object + +LOG = logging.getLogger(__name__) + + +class Tftp(Object): + __typename__ = 'tftp' + + def __init__(self, env, name, tftp_root, network): + self.name = name + self.env = env self.tftp_root = tftp_root self.network = network + + def start(self): + if not self.status(): + LOG.debug('Starting TFTP') + self.env.driver.tftp_start(self) + + def stop(self): + if self.status(): + LOG.debug('Stopping TFTP') + self.env.driver.tftp_stop(self) + + def status(self): + status = self.env.driver.tftp_status(self) + LOG.debug('TFTP status %s' % status) + return status diff --git a/fuel_agent_ci/fuel_agent_ci/objects/vm.py b/fuel_agent_ci/fuel_agent_ci/objects/vm.py index 75c7252ae0..2e7c847cef 100644 --- a/fuel_agent_ci/fuel_agent_ci/objects/vm.py +++ b/fuel_agent_ci/fuel_agent_ci/objects/vm.py @@ -12,9 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging -class Vm(object): - def __init__(self, name, boot=None): +from fuel_agent_ci.objects import Object + +LOG = logging.getLogger(__name__) + + +class Vm(Object): + __typename__ = 'vm' + + def __init__(self, env, name, boot=None): + self.env = env self.name = name self.interfaces = [] self.disks = [] @@ -36,6 +45,21 @@ class Vm(object): self.disks.append(disk) return disk + def start(self): + if not self.status(): + LOG.debug('Starting virtual machine %s' % self.name) + self.env.driver.vm_start(self) + + def stop(self): + if self.status(): + LOG.debug('Stopping virtual machine %s' % self.name) + self.env.driver.vm_stop(self) + + def status(self): + status = self.env.driver.vm_status(self) + LOG.debug('Virtual machine %s status %s' % (self.name, status)) + return status + class Interface(object): def __init__(self, mac, network): diff --git a/fuel_agent_ci/fuel_agent_ci/objects/network.py b/fuel_agent_ci/fuel_agent_ci/tests/__init__.py similarity index 76% rename from fuel_agent_ci/fuel_agent_ci/objects/network.py rename to fuel_agent_ci/fuel_agent_ci/tests/__init__.py index 0cbb49cfdd..a2bc2f3ca8 100644 --- a/fuel_agent_ci/fuel_agent_ci/objects/network.py +++ b/fuel_agent_ci/fuel_agent_ci/tests/__init__.py @@ -11,10 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -class Network(object): - def __init__(self, name, bridge, ip, forward): - self.name = name - self.bridge = bridge - self.ip = ip - self.forward = forward diff --git a/fuel_agent_ci/fuel_agent_ci/utils.py b/fuel_agent_ci/fuel_agent_ci/utils.py index c56433211d..0ec564267f 100644 --- a/fuel_agent_ci/fuel_agent_ci/utils.py +++ b/fuel_agent_ci/fuel_agent_ci/utils.py @@ -12,14 +12,60 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging +import os from random import choice +import re +import shlex import string +import subprocess + +LOG = logging.getLogger(__name__) def genmac(start=None): + LOG.debug('Generating mac address') if start is None: start = u'00:16:3e:' chars = string.digits + 'abcdef' - return start + u':'.join([ + mac = start + u':'.join([ '{0}{1}'.format(choice(chars), choice(chars)) for _ in xrange(3)]) + LOG.debug('Generated mac: %s' % mac) + return mac + +def execute(command, to_filename=None, cwd=None): + LOG.debug('Trying to execute command: %s', command) + commands = [c.strip() for c in re.split(ur'\|', command)] + env = os.environ + env['PATH'] = '/bin:/usr/bin:/sbin:/usr/sbin' + + to_file = None + if to_filename: + to_file = open(to_filename, 'wb') + + process = [] + for c in commands: + try: + # NOTE(eli): Python's shlex implementation doesn't like unicode. + # We have to convert to ascii before shlex'ing the command. + # http://bugs.python.org/issue6988 + encoded_command = c.encode('ascii') + + process.append(subprocess.Popen( + shlex.split(encoded_command), + env=env, + stdin=(process[-1].stdout if process else None), + stdout=(to_file + if (len(process) == len(commands) - 1) and to_file + else subprocess.PIPE), + stderr=(subprocess.PIPE), + cwd=cwd + )) + except OSError as e: + return (1, '', '{0}\n'.format(e)) + + if len(process) >= 2: + process[-2].stdout.close() + stdout, stderr = process[-1].communicate() + return (process[-1].returncode, stdout, stderr) diff --git a/fuel_agent_ci/requirements.txt b/fuel_agent_ci/requirements.txt index 3048ebce8b..4d2adcbd51 100644 --- a/fuel_agent_ci/requirements.txt +++ b/fuel_agent_ci/requirements.txt @@ -3,3 +3,5 @@ ipaddr>=2.1.11 libvirt-python>=1.2.5 xmlbuilder>=1.0 PyYAML>=3.11 +# pygit2>=0.20.3 +venvgit2>=0.20.3.0 diff --git a/fuel_agent_ci/samples/ci_environment.yaml b/fuel_agent_ci/samples/ci_environment.yaml index 0d6959114d..9ea1d3de72 100644 --- a/fuel_agent_ci/samples/ci_environment.yaml +++ b/fuel_agent_ci/samples/ci_environment.yaml @@ -1,31 +1,69 @@ name: "fuel_agent_ci" +envdir: "/var/tmp/fuel_agent_ci" -networks: +net: - name: "net" ip: "10.250.2.1" bridge: "ci" forward: "nat" -dhcp: - start: "10.250.2.2" - end: "10.250.2.254" - hosts: - - mac: "52:54:a5:45:65:ae" - ip: "10.250.2.20" - name: "fuel-agent-ci.domain.tld" - bootp: - file: "pxelinux.0" - network: "net" - -tftp: - tftp_root: "tftpboot" - network: "net" - -virtual_machines: +vm: - name: "vm" interfaces: - mac: "52:54:a5:45:65:ae" network: "net" disks: - size: "10240" - boot: "network" \ No newline at end of file + boot: "network" + +ssh: + - name: "vm" + host: "10.250.2.20" + user: "root" + key_filename: "ssh/id_rsa" + +repo: + - name: "fuel_agent" + url: "https://github.com/stackforge/fuel-web.git" + branch: "master" + path: "fuel_agent" + +artifact: + - name: "tftpboot" + url: "http://desktop:9090/tftpboot.tgz" + path: "tftpboot.tgz" + unpack: "tar zxf tftpboot.tgz" + clean: "rm -rf tftpboot tftpboot.tgz" + - name: "image" + url: "http://desktop:9090/image.tgz" + path: "image.tgz" + unpack: "tar zxf image.tgz" + clean: "rm -rf image image.tgz" + - name: "ssh" + url: "http://desktop:9090/ssh.tgz" + path: "ssh.tgz" + unpack: "tar zxf ssh.tgz" + clean: "rm -rf ssh ssh.tgz" + +dhcp: + - name: "dhcp" + begin: "10.250.2.2" + end: "10.250.2.254" + hosts: + - mac: "52:54:a5:45:65:ae" + ip: "10.250.2.20" + name: "fuel-agent-ci.domain.tld" + bootp: + file: "pxelinux.0" + network: "net" + +tftp: + - name: "tftp" + tftp_root: "tftpboot" + network: "net" + +http: + - name: "http" + port: "8888" + http_root: "image" + network: "net" diff --git a/fuel_agent_ci/tox.ini b/fuel_agent_ci/tox.ini index d778052691..c88e72ae90 100644 --- a/fuel_agent_ci/tox.ini +++ b/fuel_agent_ci/tox.ini @@ -17,7 +17,7 @@ downloadcache = ~/cache/pip [testenv:pep8] deps = hacking==0.7 commands = - flake8 {posargs:fuel_agent} + flake8 {posargs:fuel_agent_ci} [testenv:venv] commands = {posargs:}