diff --git a/.bzrignore b/.bzrignore index b271561a..d22b6262 100644 --- a/.bzrignore +++ b/.bzrignore @@ -13,3 +13,4 @@ CA/serial* CA/newcerts/*.pem CA/private/cakey.pem nova/vcsversion.py +*.DS_Store diff --git a/Authors b/Authors index 494e614a..a3c06dcf 100644 --- a/Authors +++ b/Authors @@ -36,6 +36,7 @@ Joshua McKenty Justin Santa Barbara Kei Masumoto Ken Pepple +Kevin L. Mitchell Koji Iida Lorin Hochstein Matt Dietz diff --git a/MANIFEST.in b/MANIFEST.in index f0a9cffb..2ceed34f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -38,3 +38,4 @@ include nova/tests/db/nova.austin.sqlite include plugins/xenapi/README include plugins/xenapi/etc/xapi.d/plugins/objectstore include plugins/xenapi/etc/xapi.d/plugins/pluginlib_nova.py +global-exclude *.pyc diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index 2bc40765..bbd60bad 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -25,7 +25,6 @@ from eventlet.green import urllib2 import exceptions import gettext -import logging import os import sys import time @@ -48,9 +47,11 @@ from nova import utils from nova import wsgi FLAGS = flags.FLAGS - flags.DEFINE_integer('ajax_console_idle_timeout', 300, 'Seconds before idle connection destroyed') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) LOG = logging.getLogger('nova.ajax_console_proxy') LOG.setLevel(logging.DEBUG) @@ -62,10 +63,16 @@ class AjaxConsoleProxy(object): def __call__(self, env, start_response): try: - req_url = '%s://%s%s?%s' % (env['wsgi.url_scheme'], - env['HTTP_HOST'], - env['PATH_INFO'], - env['QUERY_STRING']) + if 'QUERY_STRING' in env: + req_url = '%s://%s%s?%s' % (env['wsgi.url_scheme'], + env['HTTP_HOST'], + env['PATH_INFO'], + env['QUERY_STRING']) + else: + req_url = '%s://%s%s' % (env['wsgi.url_scheme'], + env['HTTP_HOST'], + env['PATH_INFO']) + if 'HTTP_REFERER' in env: auth_url = env['HTTP_REFERER'] else: @@ -130,6 +137,7 @@ class AjaxConsoleProxy(object): if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) + logging.setup() server = wsgi.Server() acp = AjaxConsoleProxy() acp.register_listeners() diff --git a/bin/nova-api b/bin/nova-api index 11176a02..14be4b84 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -36,14 +36,22 @@ gettext.install('nova', unicode=1) from nova import flags from nova import log as logging +from nova import utils from nova import version from nova import wsgi -logging.basicConfig() LOG = logging.getLogger('nova.api') -LOG.setLevel(logging.DEBUG) FLAGS = flags.FLAGS +flags.DEFINE_string('ec2_listen', "0.0.0.0", + 'IP address for EC2 API to listen') +flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') +flags.DEFINE_string('osapi_listen', "0.0.0.0", + 'IP address for OpenStack API to listen') +flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) API_ENDPOINTS = ['ec2', 'osapi'] @@ -57,21 +65,15 @@ def run_app(paste_config_file): LOG.debug(_("No paste configuration for app: %s"), api) continue LOG.debug(_("App Config: %(api)s\n%(config)r") % locals()) - wsgi.paste_config_to_flags(config, { - "verbose": FLAGS.verbose, - "%s_host" % api: config.get('host', '0.0.0.0'), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) LOG.info(_("Running %s API"), api) app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), - getattr(FLAGS, "%s_host" % api))) + apps.append((app, getattr(FLAGS, "%s_listen_port" % api), + getattr(FLAGS, "%s_listen" % api))) if len(apps) == 0: LOG.error(_("No known API applications configured in %s."), paste_config_file) return - # NOTE(todd): redo logging config, verbose could be set in paste config - logging.basicConfig() server = wsgi.Server() for app in apps: server.start(*app) @@ -79,9 +81,15 @@ def run_app(paste_config_file): if __name__ == '__main__': + utils.default_flagfile() FLAGS(sys.argv) + logging.setup() LOG.audit(_("Starting nova-api node (version %s)"), version.version_string_with_vcs()) + LOG.debug(_("Full set of FLAGS:")) + for flag in FLAGS: + flag_get = FLAGS.get(flag, None) + LOG.debug("%(flag)s : %(flag_get)s" % locals()) conf = wsgi.paste_config_file('nova-api.conf') if conf: run_app(conf) diff --git a/bin/nova-combined b/bin/nova-combined deleted file mode 100755 index 913c866b..00000000 --- a/bin/nova-combined +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Combined starter script for Nova services.""" - -import eventlet -eventlet.monkey_patch() - -import gettext -import os -import sys - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -gettext.install('nova', unicode=1) - -from nova import flags -from nova import log as logging -from nova import service -from nova import utils -from nova import wsgi - - -FLAGS = flags.FLAGS - - -if __name__ == '__main__': - utils.default_flagfile() - FLAGS(sys.argv) - logging.basicConfig() - - compute = service.Service.create(binary='nova-compute') - network = service.Service.create(binary='nova-network') - volume = service.Service.create(binary='nova-volume') - scheduler = service.Service.create(binary='nova-scheduler') - #objectstore = service.Service.create(binary='nova-objectstore') - - service.serve(compute, network, volume, scheduler) - - apps = [] - paste_config_file = wsgi.paste_config_file('nova-api.conf') - for api in ['osapi', 'ec2']: - config = wsgi.load_paste_configuration(paste_config_file, api) - if config is None: - continue - wsgi.paste_config_to_flags(config, { - "verbose": FLAGS.verbose, - "%s_host" % api: config.get('host', '0.0.0.0'), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) - app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), - getattr(FLAGS, "%s_host" % api))) - if len(apps) > 0: - logging.basicConfig() - server = wsgi.Server() - for app in apps: - server.start(*app) - server.wait() diff --git a/bin/nova-compute b/bin/nova-compute index d2d352da..95fa393b 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-console b/bin/nova-console index 802cc80b..40608b99 100755 --- a/bin/nova-console +++ b/bin/nova-console @@ -35,10 +35,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index d38ba254..3dd9de36 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -102,19 +102,10 @@ def main(): flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile) utils.default_flagfile(flagfile) argv = FLAGS(sys.argv) - logging.basicConfig() + logging.setup() interface = os.environ.get('DNSMASQ_INTERFACE', 'br0') if int(os.environ.get('TESTING', '0')): - FLAGS.fake_rabbit = True - FLAGS.network_size = 16 - FLAGS.connection_type = 'fake' - FLAGS.fake_network = True - FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' - FLAGS.num_networks = 5 - path = os.path.abspath(os.path.join(os.path.dirname(__file__), - '..', - 'nova.sqlite')) - FLAGS.sql_connection = 'sqlite:///%s' % path + from nova.tests import fake_flags action = argv[1] if action in ['add', 'del', 'old']: mac = argv[2] diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 173b39bd..bf29d9a5 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -35,6 +35,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) from nova import flags +from nova import log as logging from nova import utils from nova import wsgi from nova.api import direct @@ -44,10 +45,15 @@ from nova.compute import api as compute_api FLAGS = flags.FLAGS flags.DEFINE_integer('direct_port', 8001, 'Direct API port') flags.DEFINE_string('direct_host', '0.0.0.0', 'Direct API host') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) + if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) + logging.setup() direct.register_service('compute', compute_api.API()) direct.register_service('reflect', direct.Reflection()) diff --git a/bin/nova-import-canonical-imagestore b/bin/nova-import-canonical-imagestore index 036b41e4..404ae37f 100755 --- a/bin/nova-import-canonical-imagestore +++ b/bin/nova-import-canonical-imagestore @@ -41,6 +41,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) from nova import flags +from nova import log as logging from nova import utils from nova.objectstore import image @@ -92,6 +93,7 @@ def main(): """Main entry point.""" utils.default_flagfile() argv = FLAGS(sys.argv) + logging.setup() images = get_images() if len(argv) == 2: diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index 7dca0201..24cc9fd2 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -41,9 +41,6 @@ from nova import utils from nova import twistd from nova.compute import monitor -# TODO(todd): shouldn't this be done with flags? And what about verbose? -logging.getLogger('boto').setLevel(logging.WARN) - LOG = logging.getLogger('nova.instancemonitor') diff --git a/bin/nova-manage b/bin/nova-manage index 49246fcc..d2b4c654 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -86,8 +86,6 @@ from nova.auth import manager from nova.cloudpipe import pipelib from nova.db import migration - -logging.basicConfig() FLAGS = flags.FLAGS flags.DECLARE('fixed_range', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') @@ -95,6 +93,9 @@ flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) def param2id(object_id): @@ -586,7 +587,7 @@ class ServiceCommands(object): args: [host] [service]""" ctxt = context.get_admin_context() now = datetime.datetime.utcnow() - services = db.service_get_all(ctxt) + services = db.service_get_all(ctxt) + db.service_get_all(ctxt, True) if host: services = [s for s in services if s['host'] == host] if service: @@ -798,6 +799,7 @@ def main(): """Parse options and call the appropriate class/method.""" utils.default_flagfile() argv = FLAGS(sys.argv) + logging.setup() script_name = argv.pop(0) if len(argv) < 1: diff --git a/bin/nova-network b/bin/nova-network index 0143846a..101761ef 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-scheduler b/bin/nova-scheduler index f4c0eaed..0c205a80 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-volume b/bin/nova-volume index ad3ddc40..8dcdbc50 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/nova/adminclient.py b/nova/adminclient.py index c614b274..fc3c5c5f 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -23,6 +23,8 @@ import base64 import boto import boto.exception import httplib +import re +import string from boto.ec2.regioninfo import RegionInfo @@ -165,19 +167,20 @@ class HostInfo(object): **Fields Include** - * Disk stats - * Running Instances - * Memory stats - * CPU stats - * Network address info - * Firewall info - * Bridge and devices - + * Hostname + * Compute service status + * Volume service status + * Instance count + * Volume count """ def __init__(self, connection=None): self.connection = connection self.hostname = None + self.compute = None + self.volume = None + self.instance_count = 0 + self.volume_count = 0 def __repr__(self): return 'Host:%s' % self.hostname @@ -188,7 +191,39 @@ class HostInfo(object): # this is needed by the sax parser, so ignore the ugly name def endElement(self, name, value, connection): - setattr(self, name, value) + fixed_name = string.lower(re.sub(r'([A-Z])', r'_\1', name)) + setattr(self, fixed_name, value) + + +class Vpn(object): + """ + Information about a Vpn, as parsed through SAX + + **Fields Include** + + * instance_id + * project_id + * public_ip + * public_port + * created_at + * internal_ip + * state + """ + + def __init__(self, connection=None): + self.connection = connection + self.instance_id = None + self.project_id = None + + def __repr__(self): + return 'Vpn:%s:%s' % (self.project_id, self.instance_id) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + fixed_name = string.lower(re.sub(r'([A-Z])', r'_\1', name)) + setattr(self, fixed_name, value) class InstanceType(object): @@ -422,6 +457,16 @@ class NovaAdminClient(object): zip = self.apiconn.get_object('GenerateX509ForUser', params, UserInfo) return zip.file + def start_vpn(self, project): + """ + Starts the vpn for a user + """ + return self.apiconn.get_object('StartVpn', {'Project': project}, Vpn) + + def get_vpns(self): + """Return a list of vpn with project name""" + return self.apiconn.get_list('DescribeVpns', {}, [('item', Vpn)]) + def get_hosts(self): return self.apiconn.get_list('DescribeHosts', {}, [('item', HostInfo)]) diff --git a/nova/flags.py b/nova/flags.py index f64a62da..8cf199b2 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -160,9 +160,45 @@ class StrWrapper(object): raise KeyError(name) -FLAGS = FlagValues() -gflags.FLAGS = FLAGS -gflags.DEFINE_flag(gflags.HelpFlag(), FLAGS) +# Copied from gflags with small mods to get the naming correct. +# Originally gflags checks for the first module that is not gflags that is +# in the call chain, we want to check for the first module that is not gflags +# and not this module. +def _GetCallingModule(): + """Returns the name of the module that's calling into this module. + + We generally use this function to get the name of the module calling a + DEFINE_foo... function. + """ + # Walk down the stack to find the first globals dict that's not ours. + for depth in range(1, sys.getrecursionlimit()): + if not sys._getframe(depth).f_globals is globals(): + module_name = __GetModuleName(sys._getframe(depth).f_globals) + if module_name == 'gflags': + continue + if module_name is not None: + return module_name + raise AssertionError("No module was found") + + +# Copied from gflags because it is a private function +def __GetModuleName(globals_dict): + """Given a globals dict, returns the name of the module that defines it. + + Args: + globals_dict: A dictionary that should correspond to an environment + providing the values of the globals. + + Returns: + A string (the name of the module) or None (if the module could not + be identified. + """ + for name, module in sys.modules.iteritems(): + if getattr(module, '__dict__', None) is globals_dict: + if name == '__main__': + return sys.argv[0] + return name + return None def _wrapper(func): @@ -173,6 +209,11 @@ def _wrapper(func): return _wrapped +FLAGS = FlagValues() +gflags.FLAGS = FLAGS +gflags._GetCallingModule = _GetCallingModule + + DEFINE = _wrapper(gflags.DEFINE) DEFINE_string = _wrapper(gflags.DEFINE_string) DEFINE_integer = _wrapper(gflags.DEFINE_integer) @@ -185,8 +226,6 @@ DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) DEFINE_flag = _wrapper(gflags.DEFINE_flag) - - HelpFlag = gflags.HelpFlag HelpshortFlag = gflags.HelpshortFlag HelpXMLFlag = gflags.HelpXMLFlag @@ -285,8 +324,9 @@ DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'), DEFINE_string('logdir', None, 'output to a per-service log file in named ' 'directory') +DEFINE_string('sqlite_db', 'nova.sqlite', 'file name for sqlite') DEFINE_string('sql_connection', - 'sqlite:///$state_path/nova.sqlite', + 'sqlite:///$state_path/$sqlite_db', 'connection string for sql database') DEFINE_integer('sql_idle_timeout', 3600, diff --git a/nova/log.py b/nova/log.py index 6b201ffc..87a21ddb 100644 --- a/nova/log.py +++ b/nova/log.py @@ -54,7 +54,7 @@ flags.DEFINE_string('logging_default_format_string', 'format string to use for log messages without context') flags.DEFINE_string('logging_debug_format_suffix', - 'from %(processName)s (pid=%(process)d) %(funcName)s' + 'from (pid=%(process)d) %(funcName)s' ' %(pathname)s:%(lineno)d', 'data to append to log format when level is DEBUG') @@ -65,6 +65,7 @@ flags.DEFINE_string('logging_exception_prefix', flags.DEFINE_list('default_log_levels', ['amqplib=WARN', 'sqlalchemy=WARN', + 'boto=WARN', 'eventlet.wsgi.server=WARN'], 'list of logger=LEVEL pairs') @@ -117,7 +118,7 @@ def _get_binary_name(): return os.path.basename(inspect.stack()[-1][1]) -def get_log_file_path(binary=None): +def _get_log_file_path(binary=None): if FLAGS.logfile: return FLAGS.logfile if FLAGS.logdir: @@ -125,25 +126,6 @@ def get_log_file_path(binary=None): return '%s.log' % (os.path.join(FLAGS.logdir, binary),) -def basicConfig(): - logging.basicConfig() - for handler in logging.root.handlers: - handler.setFormatter(_formatter) - if FLAGS.verbose: - logging.root.setLevel(logging.DEBUG) - else: - logging.root.setLevel(logging.INFO) - if FLAGS.use_syslog: - syslog = SysLogHandler(address='/dev/log') - syslog.setFormatter(_formatter) - logging.root.addHandler(syslog) - logpath = get_log_file_path() - if logpath: - logfile = WatchedFileHandler(logpath) - logfile.setFormatter(_formatter) - logging.root.addHandler(logfile) - - class NovaLogger(logging.Logger): """ NovaLogger manages request context and formatting. @@ -151,23 +133,19 @@ class NovaLogger(logging.Logger): This becomes the class that is instanciated by logging.getLogger. """ def __init__(self, name, level=NOTSET): - level_name = self._get_level_from_flags(name, FLAGS) - level = globals()[level_name] logging.Logger.__init__(self, name, level) + self.setup_from_flags() - def _get_level_from_flags(self, name, FLAGS): - # if exactly "nova", or a child logger, honor the verbose flag - if (name == "nova" or name.startswith("nova.")) and FLAGS.verbose: - return 'DEBUG' + def setup_from_flags(self): + """Setup logger from flags""" + level = NOTSET for pair in FLAGS.default_log_levels: - logger, _sep, level = pair.partition('=') + logger, _sep, level_name = pair.partition('=') # NOTE(todd): if we set a.b, we want a.b.c to have the same level # (but not a.bc, so we check the dot) - if name == logger: - return level - if name.startswith(logger) and name[len(logger)] == '.': - return level - return 'INFO' + if self.name == logger or self.name.startswith("%s." % logger): + level = globals()[level_name] + self.setLevel(level) def _log(self, level, msg, args, exc_info=None, extra=None, context=None): """Extract context from any log call""" @@ -176,12 +154,12 @@ class NovaLogger(logging.Logger): if context: extra.update(_dictify_context(context)) extra.update({"nova_version": version.version_string_with_vcs()}) - logging.Logger._log(self, level, msg, args, exc_info, extra) + return logging.Logger._log(self, level, msg, args, exc_info, extra) def addHandler(self, handler): """Each handler gets our custom formatter""" handler.setFormatter(_formatter) - logging.Logger.addHandler(self, handler) + return logging.Logger.addHandler(self, handler) def audit(self, msg, *args, **kwargs): """Shortcut for our AUDIT level""" @@ -208,23 +186,6 @@ class NovaLogger(logging.Logger): self.error(message, **kwargs) -def handle_exception(type, value, tb): - logging.root.critical(str(value), exc_info=(type, value, tb)) - - -sys.excepthook = handle_exception -logging.setLoggerClass(NovaLogger) - - -class NovaRootLogger(NovaLogger): - pass - -if not isinstance(logging.root, NovaRootLogger): - logging.root = NovaRootLogger("nova.root", WARNING) - NovaLogger.root = logging.root - NovaLogger.manager.root = logging.root - - class NovaFormatter(logging.Formatter): """ A nova.context.RequestContext aware formatter configured through flags. @@ -271,8 +232,73 @@ class NovaFormatter(logging.Formatter): _formatter = NovaFormatter() +class NovaRootLogger(NovaLogger): + def __init__(self, name, level=NOTSET): + self.logpath = None + self.filelog = None + self.streamlog = StreamHandler() + self.syslog = None + NovaLogger.__init__(self, name, level) + + def setup_from_flags(self): + """Setup logger from flags""" + global _filelog + if FLAGS.use_syslog: + self.syslog = SysLogHandler(address='/dev/log') + self.addHandler(self.syslog) + elif self.syslog: + self.removeHandler(self.syslog) + logpath = _get_log_file_path() + if logpath: + self.removeHandler(self.streamlog) + if logpath != self.logpath: + self.removeHandler(self.filelog) + self.filelog = WatchedFileHandler(logpath) + self.addHandler(self.filelog) + self.logpath = logpath + else: + self.removeHandler(self.filelog) + self.addHandler(self.streamlog) + if FLAGS.verbose: + self.setLevel(DEBUG) + else: + self.setLevel(INFO) + + +def handle_exception(type, value, tb): + logging.root.critical(str(value), exc_info=(type, value, tb)) + + +def reset(): + """Resets logging handlers. Should be called if FLAGS changes.""" + for logger in NovaLogger.manager.loggerDict.itervalues(): + if isinstance(logger, NovaLogger): + logger.setup_from_flags() + + +def setup(): + """Setup nova logging.""" + if not isinstance(logging.root, NovaRootLogger): + logging._acquireLock() + for handler in logging.root.handlers: + logging.root.removeHandler(handler) + logging.root = NovaRootLogger("nova") + NovaLogger.root = logging.root + NovaLogger.manager.root = logging.root + for logger in NovaLogger.manager.loggerDict.itervalues(): + logger.root = logging.root + if isinstance(logger, logging.Logger): + NovaLogger.manager._fixupParents(logger) + NovaLogger.manager.loggerDict["nova"] = logging.root + logging._releaseLock() + sys.excepthook = handle_exception + reset() + + +root = logging.root +logging.setLoggerClass(NovaLogger) + + def audit(msg, *args, **kwargs): """Shortcut for logging to root log with sevrity 'AUDIT'.""" - if len(logging.root.handlers) == 0: - basicConfig() logging.root.log(AUDIT, msg, *args, **kwargs) diff --git a/nova/rpc.py b/nova/rpc.py index 205bb524..8fe4565d 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -91,18 +91,19 @@ class Consumer(messaging.Consumer): super(Consumer, self).__init__(*args, **kwargs) self.failed_connection = False break - except: # Catching all because carrot sucks + except Exception as e: # Catching all because carrot sucks fl_host = FLAGS.rabbit_host fl_port = FLAGS.rabbit_port fl_intv = FLAGS.rabbit_retry_interval - LOG.exception(_("AMQP server on %(fl_host)s:%(fl_port)d is" - " unreachable. Trying again in %(fl_intv)d seconds.") + LOG.error(_("AMQP server on %(fl_host)s:%(fl_port)d is" + " unreachable: %(e)s. Trying again in %(fl_intv)d" + " seconds.") % locals()) self.failed_connection = True if self.failed_connection: - LOG.exception(_("Unable to connect to AMQP server " - "after %d tries. Shutting down."), - FLAGS.rabbit_max_retries) + LOG.error(_("Unable to connect to AMQP server " + "after %d tries. Shutting down."), + FLAGS.rabbit_max_retries) sys.exit(1) def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 1097488e..cbd94947 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -29,8 +29,8 @@ FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') flags.DECLARE('fake_network', 'nova.network.manager') -FLAGS.network_size = 16 -FLAGS.num_networks = 5 +FLAGS.network_size = 8 +FLAGS.num_networks = 2 FLAGS.fake_network = True flags.DECLARE('num_shelves', 'nova.volume.driver') flags.DECLARE('blades_per_shelf', 'nova.volume.driver') @@ -39,5 +39,5 @@ FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True -FLAGS.sql_connection = 'sqlite:///nova.sqlite' +FLAGS.sqlite_db = "tests.sqlite" FLAGS.use_ipv6 = True diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index da86e6e1..5a1be08e 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -311,4 +311,5 @@ class S3APITestCase(test.TestCase): self.auth_manager.delete_user('admin') self.auth_manager.delete_project('admin') stop_listening = defer.maybeDeferred(self.listening_port.stopListening) + super(S3APITestCase, self).tearDown() return defer.DeferredList([stop_listening]) diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index fa27825c..d5c54a1c 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -20,6 +20,7 @@ import boto from boto.ec2 import regioninfo +import datetime import httplib import random import StringIO @@ -127,6 +128,28 @@ class ApiEc2TestCase(test.TestCase): self.ec2.new_http_connection(host, is_secure).AndReturn(self.http) return self.http + def test_return_valid_isoformat(self): + """ + Ensure that the ec2 api returns datetime in xs:dateTime + (which apparently isn't datetime.isoformat()) + NOTE(ken-pepple): https://bugs.launchpad.net/nova/+bug/721297 + """ + conv = apirequest._database_to_isoformat + # sqlite database representation with microseconds + time_to_convert = datetime.datetime.strptime( + "2011-02-21 20:14:10.634276", + "%Y-%m-%d %H:%M:%S.%f") + self.assertEqual( + conv(time_to_convert), + '2011-02-21T20:14:10Z') + # mysqlite database representation + time_to_convert = datetime.datetime.strptime( + "2011-02-21 19:56:18", + "%Y-%m-%d %H:%M:%S") + self.assertEqual( + conv(time_to_convert), + '2011-02-21T19:56:18Z') + def test_xmlns_version_matches_request_version(self): self.expect_http(api_version='2010-10-30') self.mox.ReplayAll() diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 35ffffb6..2a781703 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -327,15 +327,6 @@ class AuthManagerTestCase(object): class AuthManagerLdapTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' - def __init__(self, *args, **kwargs): - AuthManagerTestCase.__init__(self) - test.TestCase.__init__(self, *args, **kwargs) - import nova.auth.fakeldap as fakeldap - if FLAGS.flush_db: - LOG.info("Flushing datastore") - r = fakeldap.Store.instance() - r.flushdb() - class AuthManagerDbTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 445cc6e8..06191001 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -65,18 +65,21 @@ class CloudTestCase(test.TestCase): self.cloud = cloud.CloudController() # set up services - self.compute = service.Service.create(binary='nova-compute') - self.compute.start() - self.network = service.Service.create(binary='nova-network') - self.network.start() + self.compute = self.start_service('compute') + self.scheduter = self.start_service('scheduler') + self.network = self.start_service('network') self.manager = manager.AuthManager() self.user = self.manager.create_user('admin', 'admin', 'admin', True) self.project = self.manager.create_project('proj', 'admin', 'proj') self.context = context.RequestContext(user=self.user, project=self.project) + host = self.network.get_network_host(self.context.elevated()) def tearDown(self): + network_ref = db.project_get_network(self.context, + self.project.id) + db.network_disassociate(self.context, network_ref['id']) self.manager.delete_project(self.project) self.manager.delete_user(self.user) self.compute.kill() @@ -102,7 +105,7 @@ class CloudTestCase(test.TestCase): address = "10.10.10.10" db.floating_ip_create(self.context, {'address': address, - 'host': FLAGS.host}) + 'host': self.network.host}) self.cloud.allocate_address(self.context) self.cloud.describe_addresses(self.context) self.cloud.release_address(self.context, @@ -115,9 +118,9 @@ class CloudTestCase(test.TestCase): address = "10.10.10.10" db.floating_ip_create(self.context, {'address': address, - 'host': FLAGS.host}) + 'host': self.network.host}) self.cloud.allocate_address(self.context) - inst = db.instance_create(self.context, {'host': FLAGS.host}) + inst = db.instance_create(self.context, {'host': self.compute.host}) fixed = self.network.allocate_fixed_ip(self.context, inst['id']) ec2_id = cloud.id_to_ec2_id(inst['id']) self.cloud.associate_address(self.context, @@ -133,6 +136,22 @@ class CloudTestCase(test.TestCase): db.instance_destroy(self.context, inst['id']) db.floating_ip_destroy(self.context, address) + def test_describe_security_groups(self): + """Makes sure describe_security_groups works and filters results.""" + sec = db.security_group_create(self.context, + {'project_id': self.context.project_id, + 'name': 'test'}) + result = self.cloud.describe_security_groups(self.context) + # NOTE(vish): should have the default group as well + self.assertEqual(len(result['securityGroupInfo']), 2) + result = self.cloud.describe_security_groups(self.context, + group_name=[sec['name']]) + self.assertEqual(len(result['securityGroupInfo']), 1) + self.assertEqual( + result['securityGroupInfo'][0]['groupName'], + sec['name']) + db.security_group_destroy(self.context, sec['id']) + def test_describe_volumes(self): """Makes sure describe_volumes works and filters results.""" vol1 = db.volume_create(self.context, {}) @@ -203,27 +222,32 @@ class CloudTestCase(test.TestCase): 'instance_type': instance_type, 'max_count': max_count} rv = self.cloud.run_instances(self.context, **kwargs) + greenthread.sleep(0.3) instance_id = rv['instancesSet'][0]['instanceId'] output = self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) + instance_id=[instance_id]) self.assertEquals(b64decode(output['output']), 'FAKE CONSOLE OUTPUT') # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_ajax_console(self): + image_id = FLAGS.default_image kwargs = {'image_id': image_id} - rv = yield self.cloud.run_instances(self.context, **kwargs) + rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] - output = yield self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) - self.assertEquals(b64decode(output['output']), - 'http://fakeajaxconsole.com/?token=FAKETOKEN') + greenthread.sleep(0.3) + output = self.cloud.get_ajax_console(context=self.context, + instance_id=[instance_id]) + self.assertEquals(output['url'], + '%s/?token=FAKETOKEN' % FLAGS.ajax_console_proxy_url) # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) - rv = yield self.cloud.terminate_instances(self.context, [instance_id]) + rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_key_generation(self): result = self._create_key('test') @@ -286,70 +310,6 @@ class CloudTestCase(test.TestCase): LOG.debug(_("Terminating instance %s"), instance_id) rv = self.compute.terminate_instance(instance_id) - def test_describe_instances(self): - """Makes sure describe_instances works.""" - instance1 = db.instance_create(self.context, {'host': 'host2'}) - comp1 = db.service_create(self.context, {'host': 'host2', - 'availability_zone': 'zone1', - 'topic': "compute"}) - result = self.cloud.describe_instances(self.context) - self.assertEqual(result['reservationSet'][0] - ['instancesSet'][0] - ['placement']['availabilityZone'], 'zone1') - db.instance_destroy(self.context, instance1['id']) - db.service_destroy(self.context, comp1['id']) - - def test_instance_update_state(self): - # TODO(termie): what is this code even testing? - def instance(num): - return { - 'reservation_id': 'r-1', - 'instance_id': 'i-%s' % num, - 'image_id': 'ami-%s' % num, - 'private_dns_name': '10.0.0.%s' % num, - 'dns_name': '10.0.0%s' % num, - 'ami_launch_index': str(num), - 'instance_type': 'fake', - 'availability_zone': 'fake', - 'key_name': None, - 'kernel_id': 'fake', - 'ramdisk_id': 'fake', - 'groups': ['default'], - 'product_codes': None, - 'state': 0x01, - 'user_data': ''} - rv = self.cloud._format_describe_instances(self.context) - logging.error(str(rv)) - self.assertEqual(len(rv['reservationSet']), 0) - - # simulate launch of 5 instances - # self.cloud.instances['pending'] = {} - #for i in xrange(5): - # inst = instance(i) - # self.cloud.instances['pending'][inst['instance_id']] = inst - - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - # report 4 nodes each having 1 of the instances - #for i in xrange(4): - # self.cloud.update_state('instances', - # {('node-%s' % i): {('i-%s' % i): - # instance(i)}}) - - # one instance should be pending still - #self.assert_(len(self.cloud.instances['pending'].keys()) == 1) - - # check that the reservations collapse - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - - # check that we can get metadata for each instance - #for i in xrange(4): - # data = self.cloud.get_metadata(instance(i)['private_dns_name']) - # self.assert_(data['meta-data']['ami-id'] == 'ami-%s' % i) - @staticmethod def _fake_set_image_description(ctxt, image_id, description): from nova.objectstore import handler diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 85bf9445..49ff2441 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -21,7 +21,6 @@ Tests For Console proxy. """ import datetime -import logging from nova import context from nova import db @@ -38,7 +37,6 @@ FLAGS = flags.FLAGS class ConsoleTestCase(test.TestCase): """Test case for console proxy""" def setUp(self): - logging.getLogger().setLevel(logging.DEBUG) super(ConsoleTestCase, self).setUp() self.flags(console_driver='nova.console.fake.FakeConsoleProxy', stub_compute=True) diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index 8a74b229..b6bfab53 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -19,7 +19,6 @@ """Tests for Direct API.""" import json -import logging import webob @@ -53,6 +52,7 @@ class DirectTestCase(test.TestCase): def tearDown(self): direct.ROUTES = {} + super(DirectTestCase, self).tearDown() def test_delegated_auth(self): req = webob.Request.blank('/fake/context') diff --git a/nova/tests/test_localization.py b/nova/tests/test_localization.py index 6992773f..393d7103 100644 --- a/nova/tests/test_localization.py +++ b/nova/tests/test_localization.py @@ -15,7 +15,6 @@ # under the License. import glob -import logging import os import re import sys diff --git a/nova/tests/test_log.py b/nova/tests/test_log.py index c2c9d777..122351ff 100644 --- a/nova/tests/test_log.py +++ b/nova/tests/test_log.py @@ -1,9 +1,12 @@ import cStringIO from nova import context +from nova import flags from nova import log from nova import test +FLAGS = flags.FLAGS + def _fake_context(): return context.RequestContext(1, 1) @@ -14,15 +17,11 @@ class RootLoggerTestCase(test.TestCase): super(RootLoggerTestCase, self).setUp() self.log = log.logging.root - def tearDown(self): - super(RootLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - def test_is_nova_instance(self): self.assert_(isinstance(self.log, log.NovaLogger)) - def test_name_is_nova_root(self): - self.assertEqual("nova.root", self.log.name) + def test_name_is_nova(self): + self.assertEqual("nova", self.log.name) def test_handlers_have_nova_formatter(self): formatters = [] @@ -45,25 +44,36 @@ class RootLoggerTestCase(test.TestCase): log.audit("foo", context=_fake_context()) self.assert_(True) # didn't raise exception + def test_will_be_verbose_if_verbose_flag_set(self): + self.flags(verbose=True) + log.reset() + self.assertEqual(log.DEBUG, self.log.level) + + def test_will_not_be_verbose_if_verbose_flag_not_set(self): + self.flags(verbose=False) + log.reset() + self.assertEqual(log.INFO, self.log.level) + class LogHandlerTestCase(test.TestCase): def test_log_path_logdir(self): - self.flags(logdir='/some/path') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.flags(logdir='/some/path', logfile=None) + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') def test_log_path_logfile(self): self.flags(logfile='/some/path/foo-bar.log') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') def test_log_path_none(self): - self.assertTrue(log.get_log_file_path(binary='foo-bar') is None) + self.flags(logdir=None, logfile=None) + self.assertTrue(log._get_log_file_path(binary='foo-bar') is None) def test_log_path_logfile_overrides_logdir(self): self.flags(logdir='/some/other/path', logfile='/some/path/foo-bar.log') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') @@ -76,13 +86,15 @@ class NovaFormatterTestCase(test.TestCase): logging_debug_format_suffix="--DBG") self.log = log.logging.root self.stream = cStringIO.StringIO() - handler = log.StreamHandler(self.stream) - self.log.addHandler(handler) + self.handler = log.StreamHandler(self.stream) + self.log.addHandler(self.handler) + self.level = self.log.level self.log.setLevel(log.DEBUG) def tearDown(self): + self.log.setLevel(self.level) + self.log.removeHandler(self.handler) super(NovaFormatterTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} def test_uncontextualized_log(self): self.log.info("foo") @@ -102,30 +114,15 @@ class NovaFormatterTestCase(test.TestCase): class NovaLoggerTestCase(test.TestCase): def setUp(self): super(NovaLoggerTestCase, self).setUp() - self.flags(default_log_levels=["nova-test=AUDIT"], verbose=False) + levels = FLAGS.default_log_levels + levels.append("nova-test=AUDIT") + self.flags(default_log_levels=levels, + verbose=True) self.log = log.getLogger('nova-test') - def tearDown(self): - super(NovaLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - def test_has_level_from_flags(self): self.assertEqual(log.AUDIT, self.log.level) def test_child_log_has_level_of_parent_flag(self): l = log.getLogger('nova-test.foo') self.assertEqual(log.AUDIT, l.level) - - -class VerboseLoggerTestCase(test.TestCase): - def setUp(self): - super(VerboseLoggerTestCase, self).setUp() - self.flags(default_log_levels=["nova.test=AUDIT"], verbose=True) - self.log = log.getLogger('nova.test') - - def tearDown(self): - super(VerboseLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - - def test_will_be_verbose_if_named_nova_and_verbose_flag_set(self): - self.assertEqual(log.DEBUG, self.log.level) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 00f9323f..ce1c7721 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -42,15 +42,13 @@ class NetworkTestCase(test.TestCase): # flags in the corresponding section in nova-dhcpbridge self.flags(connection_type='fake', fake_call=True, - fake_network=True, - network_size=16, - num_networks=5) + fake_network=True) self.manager = manager.AuthManager() self.user = self.manager.create_user('netuser', 'netuser', 'netuser') self.projects = [] self.network = utils.import_object(FLAGS.network_manager) self.context = context.RequestContext(project=None, user=self.user) - for i in range(5): + for i in range(FLAGS.num_networks): name = 'project%s' % i project = self.manager.create_project(name, 'netuser', name) self.projects.append(project) @@ -117,6 +115,9 @@ class NetworkTestCase(test.TestCase): utils.to_global_ipv6( network_ref['cidr_v6'], instance_ref['mac_address'])) + self._deallocate_address(0, address) + db.instance_destroy(context.get_admin_context(), + instance_ref['id']) def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" @@ -192,7 +193,7 @@ class NetworkTestCase(test.TestCase): first = self._create_address(0) lease_ip(first) instance_ids = [] - for i in range(1, 5): + for i in range(1, FLAGS.num_networks): instance_ref = self._create_instance(i, mac=utils.generate_mac()) instance_ids.append(instance_ref['id']) address = self._create_address(i, instance_ref['id']) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 71e524bc..c4e4d148 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -249,6 +249,7 @@ class SimpleDriverTestCase(test.TestCase): def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) + super(SimpleDriverTestCase, self).tearDown() def _create_instance(self, **kwargs): """Create a test instance""" @@ -486,18 +487,8 @@ class SimpleDriverTestCase(test.TestCase): def test_doesnt_report_disabled_hosts_as_up(self): """Ensures driver doesn't find hosts before they are enabled""" - # NOTE(vish): constructing service without create method - # because we are going to use it without queue - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() - compute2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute2.start() + compute1 = self.start_service('compute', host='host1') + compute2 = self.start_service('compute', host='host2') s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') db.service_update(self.context, s1['id'], {'disabled': True}) @@ -509,18 +500,8 @@ class SimpleDriverTestCase(test.TestCase): def test_reports_enabled_hosts_as_up(self): """Ensures driver can find the hosts that are up""" - # NOTE(vish): constructing service without create method - # because we are going to use it without queue - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() - compute2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute2.start() + compute1 = self.start_service('compute', host='host1') + compute2 = self.start_service('compute', host='host2') hosts = self.scheduler.driver.hosts_up(self.context, 'compute') self.assertEqual(2, len(hosts)) compute1.kill() @@ -528,16 +509,8 @@ class SimpleDriverTestCase(test.TestCase): def test_least_busy_host_gets_instance(self): """Ensures the host with less cores gets the next one""" - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() - compute2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute2.start() + compute1 = self.start_service('compute', host='host1') + compute2 = self.start_service('compute', host='host2') instance_id1 = self._create_instance() compute1.run_instance(self.context, instance_id1) instance_id2 = self._create_instance() @@ -551,16 +524,8 @@ class SimpleDriverTestCase(test.TestCase): def test_specific_host_gets_instance(self): """Ensures if you set availability_zone it launches on that zone""" - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() - compute2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute2.start() + compute1 = self.start_service('compute', host='host1') + compute2 = self.start_service('compute', host='host2') instance_id1 = self._create_instance() compute1.run_instance(self.context, instance_id1) instance_id2 = self._create_instance(availability_zone='nova:host1') @@ -573,11 +538,7 @@ class SimpleDriverTestCase(test.TestCase): compute2.kill() def test_wont_sechedule_if_specified_host_is_down(self): - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() + compute1 = self.start_service('compute', host='host1') s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') now = datetime.datetime.utcnow() delta = datetime.timedelta(seconds=FLAGS.service_down_time * 2) @@ -592,11 +553,7 @@ class SimpleDriverTestCase(test.TestCase): compute1.kill() def test_will_schedule_on_disabled_host_if_specified(self): - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() + compute1 = self.start_service('compute', host='host1') s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') db.service_update(self.context, s1['id'], {'disabled': True}) instance_id2 = self._create_instance(availability_zone='nova:host1') @@ -608,16 +565,8 @@ class SimpleDriverTestCase(test.TestCase): def test_too_many_cores(self): """Ensures we don't go over max cores""" - compute1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute1.start() - compute2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - compute2.start() + compute1 = self.start_service('compute', host='host1') + compute2 = self.start_service('compute', host='host2') instance_ids1 = [] instance_ids2 = [] for index in xrange(FLAGS.max_cores): @@ -632,6 +581,7 @@ class SimpleDriverTestCase(test.TestCase): self.scheduler.driver.schedule_run_instance, self.context, instance_id) + db.instance_destroy(self.context, instance_id) for instance_id in instance_ids1: compute1.terminate_instance(self.context, instance_id) for instance_id in instance_ids2: @@ -641,16 +591,8 @@ class SimpleDriverTestCase(test.TestCase): def test_least_busy_host_gets_volume(self): """Ensures the host with less gigabytes gets the next one""" - volume1 = service.Service('host1', - 'nova-volume', - 'volume', - FLAGS.volume_manager) - volume1.start() - volume2 = service.Service('host2', - 'nova-volume', - 'volume', - FLAGS.volume_manager) - volume2.start() + volume1 = self.start_service('volume', host='host1') + volume2 = self.start_service('volume', host='host2') volume_id1 = self._create_volume() volume1.create_volume(self.context, volume_id1) volume_id2 = self._create_volume() @@ -664,16 +606,8 @@ class SimpleDriverTestCase(test.TestCase): def test_too_many_gigabytes(self): """Ensures we don't go over max gigabytes""" - volume1 = service.Service('host1', - 'nova-volume', - 'volume', - FLAGS.volume_manager) - volume1.start() - volume2 = service.Service('host2', - 'nova-volume', - 'volume', - FLAGS.volume_manager) - volume2.start() + volume1 = self.start_service('volume', host='host1') + volume2 = self.start_service('volume', host='host2') volume_ids1 = [] volume_ids2 = [] for index in xrange(FLAGS.max_gigabytes): @@ -893,7 +827,7 @@ class SimpleDriverTestCase(test.TestCase): {"method": 'mktmpfile'}).AndReturn(fpath) driver.rpc.call(mox.IgnoreArg(), db.queue_get_for(mox.IgnoreArg(), topic, i_ref['host']), - {"method": 'confirm_tmpfile', "args": {'path': fpath}}) + {"method": 'confirm_tmpfile', "args": {'filename': fpath}}) self.mox.ReplayAll() try: diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 91bdfcc5..f46b5950 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import libvirt import mox from xml.etree.ElementTree import fromstring as xml_to_tree @@ -23,21 +24,16 @@ from nova import context from nova import db from nova import exception from nova import flags -from nova import logging from nova import test from nova import utils from nova.api.ec2 import cloud from nova.auth import manager from nova.db.sqlalchemy import models -from nova.compute import power_state from nova.virt import libvirt_conn FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') -libvirt = None -libxml2 = None - class LibvirtConnTestCase(test.TestCase): def setUp(self): @@ -65,6 +61,7 @@ class LibvirtConnTestCase(test.TestCase): self.project = self.manager.create_project('fake', 'fake', 'fake') self.network = utils.import_object(FLAGS.network_manager) FLAGS.instances_path = '' + self.call_libvirt_dependant_setup = False test_ip = '10.11.12.13' test_instance = {'memory_kb': '1024000', @@ -76,32 +73,22 @@ class LibvirtConnTestCase(test.TestCase): 'bridge': 'br101', 'instance_type': 'm1.small'} - def _driver_dependant_test_setup(self): - """Call this method at the top of each testcase method. - - Checks if libvirt and cheetah, etc is installed. - Otherwise, skip testing.""" - + def libvirt_dependant_setup(self): + """A setup method of LibvirtConnection dependent test.""" + # try to connect libvirt. if fail, skip test. + self.call_libvirt_dependant_setup = True try: - global libvirt - global libxml2 - libvirt_conn.libvirt = __import__('libvirt') - libvirt_conn.libxml2 = __import__('libxml2') - libvirt_conn._late_load_cheetah() - libvirt = __import__('libvirt') - except ImportError, e: - logging.warn("""This test has not been done since """ - """using driver-dependent library Cheetah/libvirt/libxml2.""") - raise + libvirt.openReadOnly('qemu:///system') + except libvirt.libvirtError: + return + return libvirt_conn.get_connection(False) - # inebitable mocks for calling - obj = utils.import_object(FLAGS.firewall_driver) - fwmock = self.mox.CreateMock(obj) - self.mox.StubOutWithMock(libvirt_conn, 'utils', - use_mock_anything=True) - libvirt_conn.utils.import_object(FLAGS.firewall_driver).\ - AndReturn(fwmock) - return fwmock + def libvirt_dependant_teardown(self): + """teardown method of LibvirtConnection dependent test.""" + if self.call_libvirt_dependant_setup: + libvirt_conn.libvirt = None + libvirt_conn.libxml2 = None + self.call_libvirt_dependant_setup = False def test_xml_and_uri_no_ramdisk_no_kernel(self): instance_data = dict(self.test_instance) @@ -255,102 +242,30 @@ class LibvirtConnTestCase(test.TestCase): conn = libvirt_conn.LibvirtConnection(True) uri = conn.get_uri() self.assertEquals(uri, testuri) - - def test_get_vcpu_total(self): - """Check if get_vcpu_total returns appropriate cpu value.""" - try: - self._driver_dependant_test_setup() - except: - return - - self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) - self.assertTrue(0 < conn.get_vcpu_total()) - - def test_get_memory_mb_total(self): - """Check if get_memory_mb returns appropriate memory value.""" - try: - self._driver_dependant_test_setup() - except: - return - - self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) - self.assertTrue(0 < conn.get_memory_mb_total()) + db.instance_destroy(user_context, instance_ref['id']) def test_get_vcpu_used(self): """Check if get_local_gb_total returns appropriate disk value.""" - try: - self._driver_dependant_test_setup() - except: - return - - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, - '_conn', use_mock_anything=True) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') libvirt_conn.LibvirtConnection._conn.listDomainsID().AndReturn([1, 2]) vdmock = self.mox.CreateMock(libvirt.virDomain) - self.mox.StubOutWithMock(vdmock, "vcpus", use_mock_anything=True) + self.mox.StubOutWithMock(vdmock, "vcpus") vdmock.vcpus().AndReturn(['', [('dummycpu'), ('dummycpu')]]) vdmock.vcpus().AndReturn(['', [('dummycpu'), ('dummycpu')]]) - libvirt_conn.LibvirtConnection._conn.lookupByID(mox.IgnoreArg()).\ - AndReturn(vdmock) - libvirt_conn.LibvirtConnection._conn.lookupByID(mox.IgnoreArg()).\ - AndReturn(vdmock) + arg = mox.IgnoreArg() + libvirt_conn.LibvirtConnection._conn.lookupByID(arg).AndReturn(vdmock) + libvirt_conn.LibvirtConnection._conn.lookupByID(arg).AndReturn(vdmock) self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(conn.get_vcpu_used() == 4) - def test_get_memory_mb_used(self): - """Check if get_memory_mb returns appropriate memory value.""" - try: - self._driver_dependant_test_setup() - except: - return - - self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) - self.assertTrue(0 < conn.get_memory_mb_used()) - - def test_get_cpu_info_works_correctly(self): - """Check if get_cpu_info works correctly as expected.""" - xml = """ - x86_64 - Nehalem - Intel - - - - - - - - - - - - - - - - - - """ - - try: - self._driver_dependant_test_setup() - except: - return - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, - '_conn', use_mock_anything=True) - libvirt_conn.LibvirtConnection._conn.getCapabilities().AndReturn(xml) - - self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) - self.assertTrue(0 < len(conn.get_cpu_info())) - def test_get_cpu_info_inappropreate_xml(self): """Raise exception if given xml is inappropriate.""" + conn = self.libvirt_dependant_setup() + if not conn: + return + xml = """ x86_64 Nehalem @@ -374,16 +289,10 @@ class LibvirtConnTestCase(test.TestCase): """ - try: - self._driver_dependant_test_setup() - except: - return - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, - '_conn', use_mock_anything=True) - libvirt_conn.LibvirtConnection._conn.getCapabilities().AndReturn(xml) + self.mox.StubOutWithMock(conn._conn, 'getCapabilities') + conn._conn.getCapabilities().AndReturn(xml) self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) try: conn.get_cpu_info() except exception.Invalid, e: @@ -392,6 +301,9 @@ class LibvirtConnTestCase(test.TestCase): def test_get_cpu_info_inappropreate_xml2(self): """Raise exception if given xml is inappropriate(topology tag).""" + conn = self.libvirt_dependant_setup() + if not conn: + return xml = """ x86_64 @@ -414,16 +326,10 @@ class LibvirtConnTestCase(test.TestCase): """ - try: - self._driver_dependant_test_setup() - except: - return - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, - '_conn', use_mock_anything=True) - libvirt_conn.LibvirtConnection._conn.getCapabilities().AndReturn(xml) + self.mox.StubOutWithMock(conn._conn, 'getCapabilities') + conn._conn.getCapabilities().AndReturn(xml) self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) try: conn.get_cpu_info() except exception.Invalid, e: @@ -432,109 +338,86 @@ class LibvirtConnTestCase(test.TestCase): def test_update_available_resource_works_correctly(self): """Confirm compute_service table is updated successfully.""" - try: - self._driver_dependant_test_setup() - except: + conn = self.libvirt_dependant_setup() + if not conn: return - def dic_key_check(dic): - validkey = ['vcpus', 'memory_mb', 'local_gb', - 'vcpus_used', 'memory_mb_used', 'local_gb_used', - 'hypervisor_type', 'hypervisor_version', 'cpu_info'] - return (list(set(validkey)) == list(set(dic.keys()))) + host = 'dummy' + zone = 'dummyzone' + ctxt = context.get_admin_context() + org_path = FLAGS.instances_path = '' + FLAGS.instances_path = '.' - host = 'foo' - binary = 'nova-compute' - service_ref = {'id': 1, - 'host': host, - 'binary': binary, - 'topic': 'compute'} + service_ref = db.service_create(ctxt, + {'host': host, + 'binary': 'nova-compute', + 'topic': 'compute', + 'report_count': 0, + 'availability_zone': zone}) + conn.update_available_resource(ctxt, host) - self.mox.StubOutWithMock(db, 'service_get_all_by_topic') - db.service_get_all_by_topic(mox.IgnoreMox(), 'compute').\ - AndReturn([service_ref]) - dbmock.service_update(mox.IgnoreArg(), - service_ref['id'], - mox.Func(dic_key_check)) + service_ref = db.service_get(ctxt, service_ref['id']) + print service_ref['compute_service'] + compute_service = service_ref['compute_service'][0] + c1 = (compute_service['vcpus'] > 0) + c2 = (compute_service['memory_mb'] > 0) + c3 = (compute_service['local_gb'] > 0) + # vcpu_used is checked at test_get_vcpu_used. + c4 = (compute_service['memory_mb_used'] > 0) + c5 = (compute_service['local_gb_used'] > 0) + c6 = (len(compute_service['hypervisor_type']) > 0) + c7 = (compute_service['hypervisor_version'] > 0) - self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) - conn.update_available_resource(host) + self.assertTrue(c1 and c2 and c3 and c4 and c5 and c6 and c7) + + db.service_destroy(ctxt, service_ref['id']) + FLAGS.instances_path = org_path def test_update_resource_info_raise_exception(self): """Raise exception if no recorde found on services table.""" - try: - self._driver_dependant_test_setup() - except: - return - - host = 'foo' - binary = 'nova-compute' - dbmock = self.mox.CreateMock(db) - self.mox.StubOutWithMock(db, 'service_get_all_by_topic') - db.service_get_all_by_topic(mox.IgnoreMox(), 'compute').\ - AndRaise(exceptin.NotFound()) - - self.mox.ReplayAll() + host = 'dummy' + org_path = FLAGS.instances_path = '' + FLAGS.instances_path = '.' try: conn = libvirt_conn.LibvirtConnection(False) - conn.update_available_resource(host) + conn.update_available_resource(context.get_admin_context(), host) except exception.Invalid, e: - msg = 'Cannot insert compute manager specific info' + msg = 'Cannot update compute manager specific info' c1 = (0 <= e.message.find(msg)) - self.assertTrue(c1) + self.assertTrue(c1) + FLAGS.instances_path = org_path def test_compare_cpu_works_correctly(self): """Calling libvirt.compute_cpu() and works correctly.""" - t = {} - t['arch'] = 'x86' - t['model'] = 'model' - t['vendor'] = 'Intel' - t['topology'] = {'cores': "2", "threads": "1", "sockets": "4"} - t['features'] = ["tm"] - cpu_info = utils.dumps(t) - - try: - self._driver_dependant_test_setup() - except: + conn = self.libvirt_dependant_setup() + if not conn: return + host = 'dummy' + zone = 'dummyzone' + ctxt = context.get_admin_context() + org_path = FLAGS.instances_path = '' + FLAGS.instances_path = '.' - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, - '_conn', - use_mock_anything=True) - libvirt_conn.LibvirtConnection._conn.compareCPU(mox.IgnoreArg(), - 0).AndReturn(1) + service_ref = db.service_create(ctxt, + {'host': host, + 'binary': 'nova-compute', + 'topic': 'compute', + 'report_count': 0, + 'availability_zone': zone}) + conn.update_available_resource(ctxt, host) + service_ref = db.service_get(ctxt, service_ref['id']) + ret = conn.compare_cpu(service_ref['compute_service'][0]['cpu_info']) + self.assertTrue(ret == None) - self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) - self.assertTrue(None == conn.compare_cpu(cpu_info)) - - def test_compare_cpu_raises_exception(self): - """Libvirt-related exception occurs when calling compare_cpu().""" - t = {} - t['arch'] = 'x86' - t['model'] = 'model' - t['vendor'] = 'Intel' - t['topology'] = {'cores': "2", "threads": "1", "sockets": "4"} - t['features'] = ["tm"] - cpu_info = utils.dumps(t) - - try: - self._driver_dependant_test_setup() - except: - return - - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', - use_mock_anything=True) - libvirt_conn.LibvirtConnection._conn.compareCPU(mox.IgnoreArg(), 0).\ - AndRaise(libvirt.libvirtError('ERR')) - - self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) - self.assertRaises(libvirt.libvirtError, conn.compare_cpu, cpu_info) + db.service_destroy(ctxt, service_ref['id']) + FLAGS.instances_path = org_path def test_compare_cpu_no_compatibility(self): """Libvirt.compare_cpu() return less than 0.(no compatibility).""" + conn = self.libvirt_dependant_setup() + if not conn: + return + t = {} t['arch'] = 'x86' t['model'] = 'model' @@ -542,71 +425,66 @@ class LibvirtConnTestCase(test.TestCase): t['topology'] = {'cores': "2", "threads": "1", "sockets": "4"} t['features'] = ["tm"] cpu_info = utils.dumps(t) - - try: - self._driver_dependant_test_setup() - except: - return - - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', - use_mock_anything=True) - libvirt_conn.LibvirtConnection._conn.compareCPU(mox.IgnoreArg(), 0).\ - AndRaise(exception.Invalid('ERR')) + self.mox.StubOutWithMock(conn._conn, 'compareCPU') + conn._conn.compareCPU(mox.IgnoreArg(), 0).AndReturn(0) self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) self.assertRaises(exception.Invalid, conn.compare_cpu, cpu_info) def test_ensure_filtering_rules_for_instance_works_correctly(self): """ensure_filtering_rules_for_instance() works successfully.""" - instance_ref = models.Instance() - instance_ref.__setitem__('id', 1) - - try: - nwmock, fwmock = self._driver_dependant_test_setup() - except: + conn = self.libvirt_dependant_setup() + if not conn: return - nwmock.setup_basic_filtering(mox.IgnoreArg()) - fwmock.prepare_instance_filter(instance_ref) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', - use_mock_anything=True) + instance_ref = models.Instance() + instance_ref.__setitem__('id', 1) + fwdriver = conn.firewall_driver + + self.mox.StubOutWithMock(fwdriver, 'setup_basic_filtering') + fwdriver.setup_basic_filtering(instance_ref) + self.mox.StubOutWithMock(fwdriver, 'prepare_instance_filter') + fwdriver.prepare_instance_filter(instance_ref) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') n = 'nova-instance-%s' % instance_ref.name - libvirt_conn.LibvirtConnection._conn.nwfilterLookupByName(n) + conn._conn.nwfilterLookupByName(n) self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) conn.ensure_filtering_rules_for_instance(instance_ref) def test_ensure_filtering_rules_for_instance_timeout(self): """ensure_filtering_fules_for_instance() finishes with timeout.""" - instance_ref = models.Instance() - instance_ref.__setitem__('id', 1) - - try: - nwmock, fwmock = self._driver_dependant_test_setup() - except: + conn = self.libvirt_dependant_setup() + if not conn: return - nwmock.setup_basic_filtering(mox.IgnoreArg()) - fwmock.prepare_instance_filter(instance_ref) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', - use_mock_anything=True) + instance_ref = models.Instance() + instance_ref.__setitem__('id', 1) + fwdriver = conn.firewall_driver + + self.mox.StubOutWithMock(fwdriver, 'setup_basic_filtering') + fwdriver.setup_basic_filtering(instance_ref) + self.mox.StubOutWithMock(fwdriver, 'prepare_instance_filter') + fwdriver.prepare_instance_filter(instance_ref) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') n = 'nova-instance-%s' % instance_ref.name - for i in range(FLAGS.live_migration_timeout_sec * 2): - libvirt_conn.LibvirtConnection._conn.\ - nwfilterLookupByName(n).AndRaise(libvirt.libvirtError('ERR')) + for i in range(FLAGS.live_migration_retry_count): + conn._conn.nwfilterLookupByName(n).\ + AndRaise(libvirt.libvirtError('ERR')) self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) try: conn.ensure_filtering_rules_for_instance(instance_ref) except exception.Error, e: c1 = (0 <= e.message.find('Timeout migrating for')) - self.assertTrue(c1) + self.assertTrue(c1) def test_live_migration_works_correctly(self): """_live_migration() works as expected correctly.""" + conn = self.libvirt_dependant_setup() + if not conn: + return + class dummyCall(object): f = None @@ -615,76 +493,57 @@ class LibvirtConnTestCase(test.TestCase): i_ref = models.Instance() i_ref.__setitem__('id', 1) - i_ref.__setitem__('host', 'dummy') ctxt = context.get_admin_context() - try: - self._driver_dependant_test_setup() - except: - return - - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', - use_mock_anything=True) vdmock = self.mox.CreateMock(libvirt.virDomain) - self.mox.StubOutWithMock(vdmock, "migrateToURI", - use_mock_anything=True) - vdmock.migrateToURI(FLAGS.live_migration_uri % i_ref['host'], + self.mox.StubOutWithMock(vdmock, "migrateToURI") + vdmock.migrateToURI(FLAGS.live_migration_uri % 'dest', mox.IgnoreArg(), None, FLAGS.live_migration_bandwidth).\ AndReturn(None) - libvirt_conn.LibvirtConnection._conn.lookupByName(i_ref.name).\ - AndReturn(vdmock) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') + conn._conn.lookupByName(i_ref.name).AndReturn(vdmock) + self.mox.StubOutWithMock(libvirt_conn.utils, 'LoopingCall') libvirt_conn.utils.LoopingCall(f=None).AndReturn(dummyCall()) self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) - # Not setting post_method/recover_method in this testcase. - ret = conn._live_migration(ctxt, i_ref, i_ref['host'], '', '') + # Nothing to do with setting post_method/recover_method or not. + ret = conn._live_migration(ctxt, i_ref, 'dest', '', '') self.assertTrue(ret == None) def test_live_migration_raises_exception(self): """Confirms recover method is called when exceptions are raised.""" - i_ref = models.Instance() - i_ref.__setitem__('id', 1) - i_ref.__setitem__('host', 'dummy') - ctxt = context.get_admin_context() - - def dummy_recover_method(self, c, instance): - pass - - try: - nwmock, fwmock = self._driver_dependant_test_setup() - except: + conn = self.libvirt_dependant_setup() + if not conn: return - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', - use_mock_anything=True) + i_ref = models.Instance() + i_ref.__setitem__('id', 1) + ctxt = context.get_admin_context() + + def dummy_recover_method(c, instance, host=None): + pass + vdmock = self.mox.CreateMock(libvirt.virDomain) - self.mox.StubOutWithMock(vdmock, "migrateToURI", - use_mock_anything=True) - vdmock.migrateToURI(FLAGS.live_migration_uri % dest, mox.IgnoreArg(), + self.mox.StubOutWithMock(vdmock, "migrateToURI") + vdmock.migrateToURI(FLAGS.live_migration_uri % 'dest', + mox.IgnoreArg(), None, FLAGS.live_migration_bandwidth).\ AndRaise(libvirt.libvirtError('ERR')) - libvirt_conn.LibvirtConnection._conn.lookupByName(instance_ref.name).\ - AndReturn(vdmock) - self.mox.StubOutWithMock(db, 'instance_set_state') - db.instance_set_state(ctxt, instance_ref['id'], - power_state.RUNNING, 'running') - self.mox.StubOutWithMock(db, 'volume_update') - for v in instance_ref.volumes: - db.volume_update(ctxt, v['id'], {'status': 'in-use'}) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') + conn._conn.lookupByName(i_ref.name).AndReturn(vdmock) self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) self.assertRaises(libvirt.libvirtError, - conn._mlive_migration, - ctxt, instance_ref, dest, + conn._live_migration, + ctxt, i_ref, 'dest', '', dummy_recover_method) def tearDown(self): - super(LibvirtConnTestCase, self).tearDown() self.manager.delete_project(self.project) self.manager.delete_user(self.user) + super(LibvirtConnTestCase, self).tearDown() + self.libvirt_dependant_teardown() class IptablesFirewallTestCase(test.TestCase): @@ -841,6 +700,7 @@ class IptablesFirewallTestCase(test.TestCase): '--dports 80:81 -j ACCEPT' % security_group_chain \ in self.out_rules, "TCP port 80/81 acceptance rule wasn't added") + db.instance_destroy(admin_ctxt, instance_ref['id']) class NWFilterTestCase(test.TestCase): @@ -864,6 +724,7 @@ class NWFilterTestCase(test.TestCase): def tearDown(self): self.manager.delete_project(self.project) self.manager.delete_user(self.user) + super(NWFilterTestCase, self).tearDown() def test_cidr_rule_nwfilter_xml(self): cloud_controller = cloud.CloudController() @@ -990,3 +851,4 @@ class NWFilterTestCase(test.TestCase): self.fw.apply_instance_filter(instance) _ensure_all_called() self.teardown_security_group() + db.instance_destroy(admin_ctxt, instance_ref['id']) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 6b8efc9d..2cbe58aa 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -167,6 +167,7 @@ class XenAPIVMTestCase(test.TestCase): stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) stubs.stubout_get_this_vm_uuid(self.stubs) stubs.stubout_stream_disk(self.stubs) + stubs.stubout_is_vdi_pv(self.stubs) self.stubs.Set(VMOps, 'reset_network', reset_network) glance_stubs.stubout_glance_client(self.stubs, glance_stubs.FakeGlance) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 624995ad..4fec2bd7 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -130,6 +130,12 @@ def stubout_stream_disk(stubs): stubs.Set(vm_utils, '_stream_disk', f) +def stubout_is_vdi_pv(stubs): + def f(_1): + return False + stubs.Set(vm_utils, '_is_vdi_pv', f) + + class FakeSessionForVMTests(fake.SessionBase): """ Stubs out a XenAPISession for VM tests """ def __init__(self, uri): diff --git a/nova/twistd.py b/nova/twistd.py index 60ff7879..c07ed991 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -148,6 +148,7 @@ def WrapTwistedOptions(wrapped): options.insert(0, '') args = FLAGS(options) + logging.setup() argv = args[1:] # ignore subcommands @@ -258,7 +259,6 @@ def serve(filename): print 'usage: %s [options] [start|stop|restart]' % argv[0] sys.exit(1) - logging.basicConfig() logging.debug(_("Full set of FLAGS:")) for flag in FLAGS: logging.debug("%s : %s" % (flag, FLAGS.get(flag, None))) diff --git a/run_tests.py b/run_tests.py index 24786e8a..3c8d410e 100644 --- a/run_tests.py +++ b/run_tests.py @@ -17,26 +17,245 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Unittest runner for Nova. + +To run all tests + python run_tests.py + +To run a single test: + python run_tests.py test_compute:ComputeTestCase.test_run_terminate + +To run a single test module: + python run_tests.py test_compute + + or + + python run_tests.py api.test_wsgi + +""" + import gettext import os import unittest import sys from nose import config -from nose import result from nose import core +from nose import result from nova import log as logging +from nova.tests import fake_flags + + +class _AnsiColorizer(object): + """ + A colorizer is an object that loosely wraps around a stream, allowing + callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + """ + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm() + return curses.tigetnum("colors") > 2 + except: + raise + # guess false in case of error + return False + supported = classmethod(supported) + + def write(self, text, color): + """ + Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class _Win32Colorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + from win32console import GetStdHandle, STD_OUT_HANDLE, \ + FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \ + FOREGROUND_INTENSITY + red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, + FOREGROUND_BLUE, FOREGROUND_INTENSITY) + self.stream = stream + self.screenBuffer = GetStdHandle(STD_OUT_HANDLE) + self._colors = { + 'normal': red | green | blue, + 'red': red | bold, + 'green': green | bold, + 'blue': blue | bold, + 'yellow': red | green | bold, + 'magenta': red | blue | bold, + 'cyan': green | blue | bold, + 'white': red | green | blue | bold + } + + def supported(cls, stream=sys.stdout): + try: + import win32console + screenBuffer = win32console.GetStdHandle( + win32console.STD_OUT_HANDLE) + except ImportError: + return False + import pywintypes + try: + screenBuffer.SetConsoleTextAttribute( + win32console.FOREGROUND_RED | + win32console.FOREGROUND_GREEN | + win32console.FOREGROUND_BLUE) + except pywintypes.error: + return False + else: + return True + supported = classmethod(supported) + + def write(self, text, color): + color = self._colors[color] + self.screenBuffer.SetConsoleTextAttribute(color) + self.stream.write(text) + self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) + + +class _NullColorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + return True + supported = classmethod(supported) + + def write(self, text, color): + self.stream.write(text) class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): result.TextTestResult.__init__(self, *args, **kw) self._last_case = None + self.colorizer = None + # NOTE(vish): reset stdout for the terminal check + stdout = sys.stdout + sys.stdout = sys.__stdout__ + for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: + if colorizer.supported(): + self.colorizer = colorizer(self.stream) + break + sys.stdout = stdout def getDescription(self, test): return str(test) + # NOTE(vish): copied from unittest with edit to add color + def addSuccess(self, test): + unittest.TestResult.addSuccess(self, test) + if self.showAll: + self.colorizer.write("OK", 'green') + self.stream.writeln() + elif self.dots: + self.stream.write('.') + self.stream.flush() + + # NOTE(vish): copied from unittest with edit to add color + def addFailure(self, test, err): + unittest.TestResult.addFailure(self, test, err) + if self.showAll: + self.colorizer.write("FAIL", 'red') + self.stream.writeln() + elif self.dots: + self.stream.write('F') + self.stream.flush() + + # NOTE(vish): copied from nose with edit to add color + def addError(self, test, err): + """Overrides normal addError to add support for + errorClasses. If the exception is a registered class, the + error will be added to the list for that class, not errors. + """ + stream = getattr(self, 'stream', None) + ec, ev, tb = err + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3 compat + exc_info = self._exc_info_to_string(err) + for cls, (storage, label, isfail) in self.errorClasses.items(): + if result.isclass(ec) and issubclass(ec, cls): + if isfail: + test.passed = False + storage.append((test, exc_info)) + # Might get patched into a streamless result + if stream is not None: + if self.showAll: + message = [label] + detail = result._exception_detail(err[1]) + if detail: + message.append(detail) + stream.writeln(": ".join(message)) + elif self.dots: + stream.write(label[:1]) + return + self.errors.append((test, exc_info)) + test.passed = False + if stream is not None: + if self.showAll: + self.colorizer.write("ERROR", 'red') + self.stream.writeln() + elif self.dots: + stream.write('E') + def startTest(self, test): unittest.TestResult.startTest(self, test) current_case = test.test.__class__.__name__ @@ -60,13 +279,24 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': - logging.basicConfig() + logging.setup() + # If any argument looks like a test name but doesn't have "nova.tests" in + # front of it, automatically add that so we don't have to type as much + argv = [] + for x in sys.argv: + if x.startswith('test_'): + argv.append('nova.tests.%s' % x) + else: + argv.append(x) + + testdir = os.path.abspath(os.path.join("nova", "tests")) c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, + workingDir=testdir, plugins=core.DefaultPluginManager()) runner = NovaTestRunner(stream=c.stream, verbosity=c.verbosity, config=c) - sys.exit(not core.run(config=c, testRunner=runner)) + sys.exit(not core.run(config=c, testRunner=runner, argv=argv))