diff --git a/MANIFEST.in b/MANIFEST.in index f0a9cffb..2ceed34f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -38,3 +38,4 @@ include nova/tests/db/nova.austin.sqlite include plugins/xenapi/README include plugins/xenapi/etc/xapi.d/plugins/objectstore include plugins/xenapi/etc/xapi.d/plugins/pluginlib_nova.py +global-exclude *.pyc diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index 2bc40765..1e11c6d5 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -25,7 +25,6 @@ from eventlet.green import urllib2 import exceptions import gettext -import logging import os import sys import time @@ -48,9 +47,11 @@ from nova import utils from nova import wsgi FLAGS = flags.FLAGS - flags.DEFINE_integer('ajax_console_idle_timeout', 300, 'Seconds before idle connection destroyed') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) LOG = logging.getLogger('nova.ajax_console_proxy') LOG.setLevel(logging.DEBUG) @@ -130,6 +131,7 @@ class AjaxConsoleProxy(object): if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) + logging.setup() server = wsgi.Server() acp = AjaxConsoleProxy() acp.register_listeners() diff --git a/bin/nova-api b/bin/nova-api index 11176a02..cf140570 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -39,11 +39,18 @@ from nova import log as logging from nova import version from nova import wsgi -logging.basicConfig() LOG = logging.getLogger('nova.api') -LOG.setLevel(logging.DEBUG) FLAGS = flags.FLAGS +flags.DEFINE_string('ec2_listen', "0.0.0.0", + 'IP address for EC2 API to listen') +flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') +flags.DEFINE_string('osapi_listen', "0.0.0.0", + 'IP address for OpenStack API to listen') +flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) API_ENDPOINTS = ['ec2', 'osapi'] @@ -57,21 +64,15 @@ def run_app(paste_config_file): LOG.debug(_("No paste configuration for app: %s"), api) continue LOG.debug(_("App Config: %(api)s\n%(config)r") % locals()) - wsgi.paste_config_to_flags(config, { - "verbose": FLAGS.verbose, - "%s_host" % api: config.get('host', '0.0.0.0'), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) LOG.info(_("Running %s API"), api) app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), - getattr(FLAGS, "%s_host" % api))) + apps.append((app, getattr(FLAGS, "%s_listen_port" % api), + getattr(FLAGS, "%s_listen" % api))) if len(apps) == 0: LOG.error(_("No known API applications configured in %s."), paste_config_file) return - # NOTE(todd): redo logging config, verbose could be set in paste config - logging.basicConfig() server = wsgi.Server() for app in apps: server.start(*app) @@ -80,8 +81,13 @@ def run_app(paste_config_file): if __name__ == '__main__': FLAGS(sys.argv) + logging.setup() LOG.audit(_("Starting nova-api node (version %s)"), version.version_string_with_vcs()) + LOG.debug(_("Full set of FLAGS:")) + for flag in FLAGS: + flag_get = FLAGS.get(flag, None) + LOG.debug("%(flag)s : %(flag_get)s" % locals()) conf = wsgi.paste_config_file('nova-api.conf') if conf: run_app(conf) diff --git a/bin/nova-combined b/bin/nova-combined deleted file mode 100755 index 913c866b..00000000 --- a/bin/nova-combined +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Combined starter script for Nova services.""" - -import eventlet -eventlet.monkey_patch() - -import gettext -import os -import sys - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -gettext.install('nova', unicode=1) - -from nova import flags -from nova import log as logging -from nova import service -from nova import utils -from nova import wsgi - - -FLAGS = flags.FLAGS - - -if __name__ == '__main__': - utils.default_flagfile() - FLAGS(sys.argv) - logging.basicConfig() - - compute = service.Service.create(binary='nova-compute') - network = service.Service.create(binary='nova-network') - volume = service.Service.create(binary='nova-volume') - scheduler = service.Service.create(binary='nova-scheduler') - #objectstore = service.Service.create(binary='nova-objectstore') - - service.serve(compute, network, volume, scheduler) - - apps = [] - paste_config_file = wsgi.paste_config_file('nova-api.conf') - for api in ['osapi', 'ec2']: - config = wsgi.load_paste_configuration(paste_config_file, api) - if config is None: - continue - wsgi.paste_config_to_flags(config, { - "verbose": FLAGS.verbose, - "%s_host" % api: config.get('host', '0.0.0.0'), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) - app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), - getattr(FLAGS, "%s_host" % api))) - if len(apps) > 0: - logging.basicConfig() - server = wsgi.Server() - for app in apps: - server.start(*app) - server.wait() diff --git a/bin/nova-compute b/bin/nova-compute index d2d352da..95fa393b 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-console b/bin/nova-console index 802cc80b..40608b99 100755 --- a/bin/nova-console +++ b/bin/nova-console @@ -35,10 +35,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index d38ba254..04a1771f 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -102,7 +102,7 @@ def main(): flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile) utils.default_flagfile(flagfile) argv = FLAGS(sys.argv) - logging.basicConfig() + logging.setup() interface = os.environ.get('DNSMASQ_INTERFACE', 'br0') if int(os.environ.get('TESTING', '0')): FLAGS.fake_rabbit = True @@ -113,7 +113,9 @@ def main(): FLAGS.num_networks = 5 path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', - 'nova.sqlite')) + 'nova', + 'tests', + 'tests.sqlite')) FLAGS.sql_connection = 'sqlite:///%s' % path action = argv[1] if action in ['add', 'del', 'old']: diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 173b39bd..bf29d9a5 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -35,6 +35,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) from nova import flags +from nova import log as logging from nova import utils from nova import wsgi from nova.api import direct @@ -44,10 +45,15 @@ from nova.compute import api as compute_api FLAGS = flags.FLAGS flags.DEFINE_integer('direct_port', 8001, 'Direct API port') flags.DEFINE_string('direct_host', '0.0.0.0', 'Direct API host') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) + if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) + logging.setup() direct.register_service('compute', compute_api.API()) direct.register_service('reflect', direct.Reflection()) diff --git a/bin/nova-import-canonical-imagestore b/bin/nova-import-canonical-imagestore index 036b41e4..404ae37f 100755 --- a/bin/nova-import-canonical-imagestore +++ b/bin/nova-import-canonical-imagestore @@ -41,6 +41,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) from nova import flags +from nova import log as logging from nova import utils from nova.objectstore import image @@ -92,6 +93,7 @@ def main(): """Main entry point.""" utils.default_flagfile() argv = FLAGS(sys.argv) + logging.setup() images = get_images() if len(argv) == 2: diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index 7dca0201..24cc9fd2 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -41,9 +41,6 @@ from nova import utils from nova import twistd from nova.compute import monitor -# TODO(todd): shouldn't this be done with flags? And what about verbose? -logging.getLogger('boto').setLevel(logging.WARN) - LOG = logging.getLogger('nova.instancemonitor') diff --git a/bin/nova-manage b/bin/nova-manage index 8ecf626a..9e2e442e 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -87,8 +87,6 @@ from nova.cloudpipe import pipelib from nova.compute import instance_types from nova.db import migration - -logging.basicConfig() FLAGS = flags.FLAGS flags.DECLARE('fixed_range', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') @@ -96,6 +94,9 @@ flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) def param2id(object_id): @@ -808,6 +809,7 @@ def main(): """Parse options and call the appropriate class/method.""" utils.default_flagfile() argv = FLAGS(sys.argv) + logging.setup() script_name = argv.pop(0) if len(argv) < 1: diff --git a/bin/nova-network b/bin/nova-network index 0143846a..101761ef 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-scheduler b/bin/nova-scheduler index f4c0eaed..0c205a80 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-volume b/bin/nova-volume index ad3ddc40..8dcdbc50 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/nova/flags.py b/nova/flags.py index f64a62da..24bca0ca 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -160,9 +160,45 @@ class StrWrapper(object): raise KeyError(name) -FLAGS = FlagValues() -gflags.FLAGS = FLAGS -gflags.DEFINE_flag(gflags.HelpFlag(), FLAGS) +# Copied from gflags with small mods to get the naming correct. +# Originally gflags checks for the first module that is not gflags that is +# in the call chain, we want to check for the first module that is not gflags +# and not this module. +def _GetCallingModule(): + """Returns the name of the module that's calling into this module. + + We generally use this function to get the name of the module calling a + DEFINE_foo... function. + """ + # Walk down the stack to find the first globals dict that's not ours. + for depth in range(1, sys.getrecursionlimit()): + if not sys._getframe(depth).f_globals is globals(): + module_name = __GetModuleName(sys._getframe(depth).f_globals) + if module_name == 'gflags': + continue + if module_name is not None: + return module_name + raise AssertionError("No module was found") + + +# Copied from gflags because it is a private function +def __GetModuleName(globals_dict): + """Given a globals dict, returns the name of the module that defines it. + + Args: + globals_dict: A dictionary that should correspond to an environment + providing the values of the globals. + + Returns: + A string (the name of the module) or None (if the module could not + be identified. + """ + for name, module in sys.modules.iteritems(): + if getattr(module, '__dict__', None) is globals_dict: + if name == '__main__': + return sys.argv[0] + return name + return None def _wrapper(func): @@ -173,6 +209,11 @@ def _wrapper(func): return _wrapped +FLAGS = FlagValues() +gflags.FLAGS = FLAGS +gflags._GetCallingModule = _GetCallingModule + + DEFINE = _wrapper(gflags.DEFINE) DEFINE_string = _wrapper(gflags.DEFINE_string) DEFINE_integer = _wrapper(gflags.DEFINE_integer) @@ -185,8 +226,6 @@ DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) DEFINE_flag = _wrapper(gflags.DEFINE_flag) - - HelpFlag = gflags.HelpFlag HelpshortFlag = gflags.HelpshortFlag HelpXMLFlag = gflags.HelpXMLFlag diff --git a/nova/log.py b/nova/log.py index 6b201ffc..591d26c6 100644 --- a/nova/log.py +++ b/nova/log.py @@ -65,6 +65,7 @@ flags.DEFINE_string('logging_exception_prefix', flags.DEFINE_list('default_log_levels', ['amqplib=WARN', 'sqlalchemy=WARN', + 'boto=WARN', 'eventlet.wsgi.server=WARN'], 'list of logger=LEVEL pairs') @@ -117,7 +118,7 @@ def _get_binary_name(): return os.path.basename(inspect.stack()[-1][1]) -def get_log_file_path(binary=None): +def _get_log_file_path(binary=None): if FLAGS.logfile: return FLAGS.logfile if FLAGS.logdir: @@ -125,25 +126,6 @@ def get_log_file_path(binary=None): return '%s.log' % (os.path.join(FLAGS.logdir, binary),) -def basicConfig(): - logging.basicConfig() - for handler in logging.root.handlers: - handler.setFormatter(_formatter) - if FLAGS.verbose: - logging.root.setLevel(logging.DEBUG) - else: - logging.root.setLevel(logging.INFO) - if FLAGS.use_syslog: - syslog = SysLogHandler(address='/dev/log') - syslog.setFormatter(_formatter) - logging.root.addHandler(syslog) - logpath = get_log_file_path() - if logpath: - logfile = WatchedFileHandler(logpath) - logfile.setFormatter(_formatter) - logging.root.addHandler(logfile) - - class NovaLogger(logging.Logger): """ NovaLogger manages request context and formatting. @@ -151,23 +133,19 @@ class NovaLogger(logging.Logger): This becomes the class that is instanciated by logging.getLogger. """ def __init__(self, name, level=NOTSET): - level_name = self._get_level_from_flags(name, FLAGS) - level = globals()[level_name] logging.Logger.__init__(self, name, level) + self.setup_from_flags() - def _get_level_from_flags(self, name, FLAGS): - # if exactly "nova", or a child logger, honor the verbose flag - if (name == "nova" or name.startswith("nova.")) and FLAGS.verbose: - return 'DEBUG' + def setup_from_flags(self): + """Setup logger from flags""" + level = NOTSET for pair in FLAGS.default_log_levels: - logger, _sep, level = pair.partition('=') + logger, _sep, level_name = pair.partition('=') # NOTE(todd): if we set a.b, we want a.b.c to have the same level # (but not a.bc, so we check the dot) - if name == logger: - return level - if name.startswith(logger) and name[len(logger)] == '.': - return level - return 'INFO' + if self.name == logger or self.name.startswith("%s." % logger): + level = globals()[level_name] + self.setLevel(level) def _log(self, level, msg, args, exc_info=None, extra=None, context=None): """Extract context from any log call""" @@ -176,12 +154,12 @@ class NovaLogger(logging.Logger): if context: extra.update(_dictify_context(context)) extra.update({"nova_version": version.version_string_with_vcs()}) - logging.Logger._log(self, level, msg, args, exc_info, extra) + return logging.Logger._log(self, level, msg, args, exc_info, extra) def addHandler(self, handler): """Each handler gets our custom formatter""" handler.setFormatter(_formatter) - logging.Logger.addHandler(self, handler) + return logging.Logger.addHandler(self, handler) def audit(self, msg, *args, **kwargs): """Shortcut for our AUDIT level""" @@ -208,23 +186,6 @@ class NovaLogger(logging.Logger): self.error(message, **kwargs) -def handle_exception(type, value, tb): - logging.root.critical(str(value), exc_info=(type, value, tb)) - - -sys.excepthook = handle_exception -logging.setLoggerClass(NovaLogger) - - -class NovaRootLogger(NovaLogger): - pass - -if not isinstance(logging.root, NovaRootLogger): - logging.root = NovaRootLogger("nova.root", WARNING) - NovaLogger.root = logging.root - NovaLogger.manager.root = logging.root - - class NovaFormatter(logging.Formatter): """ A nova.context.RequestContext aware formatter configured through flags. @@ -271,8 +232,73 @@ class NovaFormatter(logging.Formatter): _formatter = NovaFormatter() +class NovaRootLogger(NovaLogger): + def __init__(self, name, level=NOTSET): + self.logpath = None + self.filelog = None + self.streamlog = StreamHandler() + self.syslog = None + NovaLogger.__init__(self, name, level) + + def setup_from_flags(self): + """Setup logger from flags""" + global _filelog + if FLAGS.use_syslog: + self.syslog = SysLogHandler(address='/dev/log') + self.addHandler(self.syslog) + elif self.syslog: + self.removeHandler(self.syslog) + logpath = _get_log_file_path() + if logpath: + self.removeHandler(self.streamlog) + if logpath != self.logpath: + self.removeHandler(self.filelog) + self.filelog = WatchedFileHandler(logpath) + self.addHandler(self.filelog) + self.logpath = logpath + else: + self.removeHandler(self.filelog) + self.addHandler(self.streamlog) + if FLAGS.verbose: + self.setLevel(DEBUG) + else: + self.setLevel(INFO) + + +def handle_exception(type, value, tb): + logging.root.critical(str(value), exc_info=(type, value, tb)) + + +def reset(): + """Resets logging handlers. Should be called if FLAGS changes.""" + for logger in NovaLogger.manager.loggerDict.itervalues(): + if isinstance(logger, NovaLogger): + logger.setup_from_flags() + + +def setup(): + """Setup nova logging.""" + if not isinstance(logging.root, NovaRootLogger): + logging._acquireLock() + for handler in logging.root.handlers: + logging.root.removeHandler(handler) + logging.root = NovaRootLogger("nova") + NovaLogger.root = logging.root + NovaLogger.manager.root = logging.root + for logger in NovaLogger.manager.loggerDict.itervalues(): + logger.root = logging.root + if isinstance(logger, logging.Logger): + NovaLogger.manager._fixupParents(logger) + NovaLogger.manager.loggerDict["nova"] = logging.root + logging._releaseLock() + sys.excepthook = handle_exception + reset() + + +root = logging.root +logging.setLoggerClass(NovaLogger) + + def audit(msg, *args, **kwargs): """Shortcut for logging to root log with sevrity 'AUDIT'.""" - if len(logging.root.handlers) == 0: - basicConfig() logging.root.log(AUDIT, msg, *args, **kwargs) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 1097488e..2b191940 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -39,5 +39,5 @@ FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True -FLAGS.sql_connection = 'sqlite:///nova.sqlite' +FLAGS.sql_connection = 'sqlite:///tests.sqlite' FLAGS.use_ipv6 = True diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index fa27825c..d5c54a1c 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -20,6 +20,7 @@ import boto from boto.ec2 import regioninfo +import datetime import httplib import random import StringIO @@ -127,6 +128,28 @@ class ApiEc2TestCase(test.TestCase): self.ec2.new_http_connection(host, is_secure).AndReturn(self.http) return self.http + def test_return_valid_isoformat(self): + """ + Ensure that the ec2 api returns datetime in xs:dateTime + (which apparently isn't datetime.isoformat()) + NOTE(ken-pepple): https://bugs.launchpad.net/nova/+bug/721297 + """ + conv = apirequest._database_to_isoformat + # sqlite database representation with microseconds + time_to_convert = datetime.datetime.strptime( + "2011-02-21 20:14:10.634276", + "%Y-%m-%d %H:%M:%S.%f") + self.assertEqual( + conv(time_to_convert), + '2011-02-21T20:14:10Z') + # mysqlite database representation + time_to_convert = datetime.datetime.strptime( + "2011-02-21 19:56:18", + "%Y-%m-%d %H:%M:%S") + self.assertEqual( + conv(time_to_convert), + '2011-02-21T19:56:18Z') + def test_xmlns_version_matches_request_version(self): self.expect_http(api_version='2010-10-30') self.mox.ReplayAll() diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 35ffffb6..2a781703 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -327,15 +327,6 @@ class AuthManagerTestCase(object): class AuthManagerLdapTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' - def __init__(self, *args, **kwargs): - AuthManagerTestCase.__init__(self) - test.TestCase.__init__(self, *args, **kwargs) - import nova.auth.fakeldap as fakeldap - if FLAGS.flush_db: - LOG.info("Flushing datastore") - r = fakeldap.Store.instance() - r.flushdb() - class AuthManagerDbTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 1824d24b..06191001 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -66,6 +66,7 @@ class CloudTestCase(test.TestCase): # set up services self.compute = self.start_service('compute') + self.scheduter = self.start_service('scheduler') self.network = self.start_service('network') self.manager = manager.AuthManager() @@ -73,8 +74,12 @@ class CloudTestCase(test.TestCase): self.project = self.manager.create_project('proj', 'admin', 'proj') self.context = context.RequestContext(user=self.user, project=self.project) + host = self.network.get_network_host(self.context.elevated()) def tearDown(self): + network_ref = db.project_get_network(self.context, + self.project.id) + db.network_disassociate(self.context, network_ref['id']) self.manager.delete_project(self.project) self.manager.delete_user(self.user) self.compute.kill() @@ -131,6 +136,22 @@ class CloudTestCase(test.TestCase): db.instance_destroy(self.context, inst['id']) db.floating_ip_destroy(self.context, address) + def test_describe_security_groups(self): + """Makes sure describe_security_groups works and filters results.""" + sec = db.security_group_create(self.context, + {'project_id': self.context.project_id, + 'name': 'test'}) + result = self.cloud.describe_security_groups(self.context) + # NOTE(vish): should have the default group as well + self.assertEqual(len(result['securityGroupInfo']), 2) + result = self.cloud.describe_security_groups(self.context, + group_name=[sec['name']]) + self.assertEqual(len(result['securityGroupInfo']), 1) + self.assertEqual( + result['securityGroupInfo'][0]['groupName'], + sec['name']) + db.security_group_destroy(self.context, sec['id']) + def test_describe_volumes(self): """Makes sure describe_volumes works and filters results.""" vol1 = db.volume_create(self.context, {}) @@ -201,27 +222,32 @@ class CloudTestCase(test.TestCase): 'instance_type': instance_type, 'max_count': max_count} rv = self.cloud.run_instances(self.context, **kwargs) + greenthread.sleep(0.3) instance_id = rv['instancesSet'][0]['instanceId'] output = self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) + instance_id=[instance_id]) self.assertEquals(b64decode(output['output']), 'FAKE CONSOLE OUTPUT') # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_ajax_console(self): + image_id = FLAGS.default_image kwargs = {'image_id': image_id} - rv = yield self.cloud.run_instances(self.context, **kwargs) + rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] - output = yield self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) - self.assertEquals(b64decode(output['output']), - 'http://fakeajaxconsole.com/?token=FAKETOKEN') + greenthread.sleep(0.3) + output = self.cloud.get_ajax_console(context=self.context, + instance_id=[instance_id]) + self.assertEquals(output['url'], + '%s/?token=FAKETOKEN' % FLAGS.ajax_console_proxy_url) # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) - rv = yield self.cloud.terminate_instances(self.context, [instance_id]) + rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_key_generation(self): result = self._create_key('test') @@ -284,70 +310,6 @@ class CloudTestCase(test.TestCase): LOG.debug(_("Terminating instance %s"), instance_id) rv = self.compute.terminate_instance(instance_id) - def test_describe_instances(self): - """Makes sure describe_instances works.""" - instance1 = db.instance_create(self.context, {'host': 'host2'}) - comp1 = db.service_create(self.context, {'host': 'host2', - 'availability_zone': 'zone1', - 'topic': "compute"}) - result = self.cloud.describe_instances(self.context) - self.assertEqual(result['reservationSet'][0] - ['instancesSet'][0] - ['placement']['availabilityZone'], 'zone1') - db.instance_destroy(self.context, instance1['id']) - db.service_destroy(self.context, comp1['id']) - - def test_instance_update_state(self): - # TODO(termie): what is this code even testing? - def instance(num): - return { - 'reservation_id': 'r-1', - 'instance_id': 'i-%s' % num, - 'image_id': 'ami-%s' % num, - 'private_dns_name': '10.0.0.%s' % num, - 'dns_name': '10.0.0%s' % num, - 'ami_launch_index': str(num), - 'instance_type': 'fake', - 'availability_zone': 'fake', - 'key_name': None, - 'kernel_id': 'fake', - 'ramdisk_id': 'fake', - 'groups': ['default'], - 'product_codes': None, - 'state': 0x01, - 'user_data': ''} - rv = self.cloud._format_describe_instances(self.context) - logging.error(str(rv)) - self.assertEqual(len(rv['reservationSet']), 0) - - # simulate launch of 5 instances - # self.cloud.instances['pending'] = {} - #for i in xrange(5): - # inst = instance(i) - # self.cloud.instances['pending'][inst['instance_id']] = inst - - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - # report 4 nodes each having 1 of the instances - #for i in xrange(4): - # self.cloud.update_state('instances', - # {('node-%s' % i): {('i-%s' % i): - # instance(i)}}) - - # one instance should be pending still - #self.assert_(len(self.cloud.instances['pending'].keys()) == 1) - - # check that the reservations collapse - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - - # check that we can get metadata for each instance - #for i in xrange(4): - # data = self.cloud.get_metadata(instance(i)['private_dns_name']) - # self.assert_(data['meta-data']['ami-id'] == 'ami-%s' % i) - @staticmethod def _fake_set_image_description(ctxt, image_id, description): from nova.objectstore import handler diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 85bf9445..49ff2441 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -21,7 +21,6 @@ Tests For Console proxy. """ import datetime -import logging from nova import context from nova import db @@ -38,7 +37,6 @@ FLAGS = flags.FLAGS class ConsoleTestCase(test.TestCase): """Test case for console proxy""" def setUp(self): - logging.getLogger().setLevel(logging.DEBUG) super(ConsoleTestCase, self).setUp() self.flags(console_driver='nova.console.fake.FakeConsoleProxy', stub_compute=True) diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index 8a74b229..7656f539 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -19,7 +19,6 @@ """Tests for Direct API.""" import json -import logging import webob diff --git a/nova/tests/test_localization.py b/nova/tests/test_localization.py index 6992773f..393d7103 100644 --- a/nova/tests/test_localization.py +++ b/nova/tests/test_localization.py @@ -15,7 +15,6 @@ # under the License. import glob -import logging import os import re import sys diff --git a/nova/tests/test_log.py b/nova/tests/test_log.py index c2c9d777..122351ff 100644 --- a/nova/tests/test_log.py +++ b/nova/tests/test_log.py @@ -1,9 +1,12 @@ import cStringIO from nova import context +from nova import flags from nova import log from nova import test +FLAGS = flags.FLAGS + def _fake_context(): return context.RequestContext(1, 1) @@ -14,15 +17,11 @@ class RootLoggerTestCase(test.TestCase): super(RootLoggerTestCase, self).setUp() self.log = log.logging.root - def tearDown(self): - super(RootLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - def test_is_nova_instance(self): self.assert_(isinstance(self.log, log.NovaLogger)) - def test_name_is_nova_root(self): - self.assertEqual("nova.root", self.log.name) + def test_name_is_nova(self): + self.assertEqual("nova", self.log.name) def test_handlers_have_nova_formatter(self): formatters = [] @@ -45,25 +44,36 @@ class RootLoggerTestCase(test.TestCase): log.audit("foo", context=_fake_context()) self.assert_(True) # didn't raise exception + def test_will_be_verbose_if_verbose_flag_set(self): + self.flags(verbose=True) + log.reset() + self.assertEqual(log.DEBUG, self.log.level) + + def test_will_not_be_verbose_if_verbose_flag_not_set(self): + self.flags(verbose=False) + log.reset() + self.assertEqual(log.INFO, self.log.level) + class LogHandlerTestCase(test.TestCase): def test_log_path_logdir(self): - self.flags(logdir='/some/path') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.flags(logdir='/some/path', logfile=None) + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') def test_log_path_logfile(self): self.flags(logfile='/some/path/foo-bar.log') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') def test_log_path_none(self): - self.assertTrue(log.get_log_file_path(binary='foo-bar') is None) + self.flags(logdir=None, logfile=None) + self.assertTrue(log._get_log_file_path(binary='foo-bar') is None) def test_log_path_logfile_overrides_logdir(self): self.flags(logdir='/some/other/path', logfile='/some/path/foo-bar.log') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') @@ -76,13 +86,15 @@ class NovaFormatterTestCase(test.TestCase): logging_debug_format_suffix="--DBG") self.log = log.logging.root self.stream = cStringIO.StringIO() - handler = log.StreamHandler(self.stream) - self.log.addHandler(handler) + self.handler = log.StreamHandler(self.stream) + self.log.addHandler(self.handler) + self.level = self.log.level self.log.setLevel(log.DEBUG) def tearDown(self): + self.log.setLevel(self.level) + self.log.removeHandler(self.handler) super(NovaFormatterTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} def test_uncontextualized_log(self): self.log.info("foo") @@ -102,30 +114,15 @@ class NovaFormatterTestCase(test.TestCase): class NovaLoggerTestCase(test.TestCase): def setUp(self): super(NovaLoggerTestCase, self).setUp() - self.flags(default_log_levels=["nova-test=AUDIT"], verbose=False) + levels = FLAGS.default_log_levels + levels.append("nova-test=AUDIT") + self.flags(default_log_levels=levels, + verbose=True) self.log = log.getLogger('nova-test') - def tearDown(self): - super(NovaLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - def test_has_level_from_flags(self): self.assertEqual(log.AUDIT, self.log.level) def test_child_log_has_level_of_parent_flag(self): l = log.getLogger('nova-test.foo') self.assertEqual(log.AUDIT, l.level) - - -class VerboseLoggerTestCase(test.TestCase): - def setUp(self): - super(VerboseLoggerTestCase, self).setUp() - self.flags(default_log_levels=["nova.test=AUDIT"], verbose=True) - self.log = log.getLogger('nova.test') - - def tearDown(self): - super(VerboseLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - - def test_will_be_verbose_if_named_nova_and_verbose_flag_set(self): - self.assertEqual(log.DEBUG, self.log.level) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 00f9323f..53cfea27 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -117,6 +117,9 @@ class NetworkTestCase(test.TestCase): utils.to_global_ipv6( network_ref['cidr_v6'], instance_ref['mac_address'])) + self._deallocate_address(0, address) + db.instance_destroy(context.get_admin_context(), + instance_ref['id']) def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 25017007..b6888c4d 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -150,6 +150,7 @@ class SimpleDriverTestCase(test.TestCase): def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) + super(SimpleDriverTestCase, self).tearDown() def _create_instance(self, **kwargs): """Create a test instance""" @@ -270,6 +271,7 @@ class SimpleDriverTestCase(test.TestCase): self.scheduler.driver.schedule_run_instance, self.context, instance_id) + db.instance_destroy(self.context, instance_id) for instance_id in instance_ids1: compute1.terminate_instance(self.context, instance_id) for instance_id in instance_ids2: diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 6e5a0114..5b3247df 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -204,6 +204,7 @@ class LibvirtConnTestCase(test.TestCase): conn = libvirt_conn.LibvirtConnection(True) uri = conn.get_uri() self.assertEquals(uri, testuri) + db.instance_destroy(user_context, instance_ref['id']) def tearDown(self): super(LibvirtConnTestCase, self).tearDown() @@ -365,6 +366,7 @@ class IptablesFirewallTestCase(test.TestCase): '--dports 80:81 -j ACCEPT' % security_group_chain \ in self.out_rules, "TCP port 80/81 acceptance rule wasn't added") + db.instance_destroy(admin_ctxt, instance_ref['id']) class NWFilterTestCase(test.TestCase): @@ -514,3 +516,4 @@ class NWFilterTestCase(test.TestCase): self.fw.apply_instance_filter(instance) _ensure_all_called() self.teardown_security_group() + db.instance_destroy(admin_ctxt, instance_ref['id']) diff --git a/nova/twistd.py b/nova/twistd.py index 60ff7879..c07ed991 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -148,6 +148,7 @@ def WrapTwistedOptions(wrapped): options.insert(0, '') args = FLAGS(options) + logging.setup() argv = args[1:] # ignore subcommands @@ -258,7 +259,6 @@ def serve(filename): print 'usage: %s [options] [start|stop|restart]' % argv[0] sys.exit(1) - logging.basicConfig() logging.debug(_("Full set of FLAGS:")) for flag in FLAGS: logging.debug("%s : %s" % (flag, FLAGS.get(flag, None))) diff --git a/run_tests.py b/run_tests.py index 24786e8a..8025548e 100644 --- a/run_tests.py +++ b/run_tests.py @@ -17,6 +17,44 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Unittest runner for Nova. + +To run all tests + python run_tests.py + +To run a single test: + python run_tests.py test_compute:ComputeTestCase.test_run_terminate + +To run a single test module: + python run_tests.py test_compute + + or + + python run_tests.py api.test_wsgi + +""" + import gettext import os import unittest @@ -27,16 +65,197 @@ from nose import result from nose import core from nova import log as logging +from nova.tests import fake_flags + + +class _AnsiColorizer(object): + """ + A colorizer is an object that loosely wraps around a stream, allowing + callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + """ + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm() + return curses.tigetnum("colors") > 2 + except: + raise + # guess false in case of error + return False + supported = classmethod(supported) + + def write(self, text, color): + """ + Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class _Win32Colorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + from win32console import GetStdHandle, STD_OUT_HANDLE, \ + FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \ + FOREGROUND_INTENSITY + red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, + FOREGROUND_BLUE, FOREGROUND_INTENSITY) + self.stream = stream + self.screenBuffer = GetStdHandle(STD_OUT_HANDLE) + self._colors = { + 'normal': red | green | blue, + 'red': red | bold, + 'green': green | bold, + 'blue': blue | bold, + 'yellow': red | green | bold, + 'magenta': red | blue | bold, + 'cyan': green | blue | bold, + 'white': red | green | blue | bold + } + + def supported(cls, stream=sys.stdout): + try: + import win32console + screenBuffer = win32console.GetStdHandle( + win32console.STD_OUT_HANDLE) + except ImportError: + return False + import pywintypes + try: + screenBuffer.SetConsoleTextAttribute( + win32console.FOREGROUND_RED | + win32console.FOREGROUND_GREEN | + win32console.FOREGROUND_BLUE) + except pywintypes.error: + return False + else: + return True + supported = classmethod(supported) + + def write(self, text, color): + color = self._colors[color] + self.screenBuffer.SetConsoleTextAttribute(color) + self.stream.write(text) + self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) + + +class _NullColorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + return True + supported = classmethod(supported) + + def write(self, text, color): + self.stream.write(text) class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): result.TextTestResult.__init__(self, *args, **kw) self._last_case = None + self.colorizer = None + # NOTE(vish): reset stdout for the terminal check + stdout = sys.stdout + sys.stdout = sys.__stdout__ + for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: + if colorizer.supported(): + self.colorizer = colorizer(self.stream) + break + sys.stdout = stdout def getDescription(self, test): return str(test) + # NOTE(vish): copied from unittest with edit to add color + def addSuccess(self, test): + unittest.TestResult.addSuccess(self, test) + if self.showAll: + self.colorizer.write("OK", 'green') + self.stream.writeln() + elif self.dots: + self.stream.write('.') + self.stream.flush() + + # NOTE(vish): copied from unittest with edit to add color + def addFailure(self, test, err): + unittest.TestResult.addFailure(self, test, err) + if self.showAll: + self.colorizer.write("FAIL", 'red') + self.stream.writeln() + elif self.dots: + self.stream.write('F') + self.stream.flush() + + # NOTE(vish): copied from nose with edit to add color + def addError(self, test, err): + """Overrides normal addError to add support for + errorClasses. If the exception is a registered class, the + error will be added to the list for that class, not errors. + """ + stream = getattr(self, 'stream', None) + ec, ev, tb = err + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3 compat + exc_info = self._exc_info_to_string(err) + for cls, (storage, label, isfail) in self.errorClasses.items(): + if result.isclass(ec) and issubclass(ec, cls): + if isfail: + test.passed = False + storage.append((test, exc_info)) + # Might get patched into a streamless result + if stream is not None: + if self.showAll: + message = [label] + detail = result._exception_detail(err[1]) + if detail: + message.append(detail) + stream.writeln(": ".join(message)) + elif self.dots: + stream.write(label[:1]) + return + self.errors.append((test, exc_info)) + test.passed = False + if stream is not None: + if self.showAll: + self.colorizer.write("ERROR", 'red') + self.stream.writeln() + elif self.dots: + stream.write('E') + def startTest(self, test): unittest.TestResult.startTest(self, test) current_case = test.test.__class__.__name__ @@ -60,13 +279,27 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': - logging.basicConfig() + logging.setup() + testdir = os.path.abspath(os.path.join("nova", "tests")) + testdb = os.path.join(testdir, "tests.sqlite") + if os.path.exists(testdb): + os.unlink(testdb) + # If any argument looks like a test name but doesn't have "nova.tests" in + # front of it, automatically add that so we don't have to type as much + argv = [] + for x in sys.argv: + if x.startswith('test_'): + argv.append('nova.tests.%s' % x) + else: + argv.append(x) + c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, + workingDir=testdir, plugins=core.DefaultPluginManager()) runner = NovaTestRunner(stream=c.stream, verbosity=c.verbosity, config=c) - sys.exit(not core.run(config=c, testRunner=runner)) + sys.exit(not core.run(config=c, testRunner=runner, argv=argv))