From 57b28c1f64c25d54354412837c14dc905ac241c2 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 2 Feb 2011 13:13:10 -0800 Subject: [PATCH 01/15] some updates to HACKING to describe the docstrings --- HACKING | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/HACKING b/HACKING index 3af2381b..4e0dfc83 100644 --- a/HACKING +++ b/HACKING @@ -47,3 +47,18 @@ Human Alphabetical Order Examples from nova.auth import users from nova.endpoint import api from nova.endpoint import cloud + +Docstrings +---------- + """Summary of the function, class or method, less than 80 characters. + + New paragraph after newline that explains in more detail any general + information about the function, class or method. After this, if defining + parameters and return types use the Sphinx format. After that an extra + newline then close the quotations. + + :param foo: the foo parameter + :param bar: the bar parameter + :rtype: the return type + + """ From c5e7d2bbea5ea3bf22908db7046f4493d0f67609 Mon Sep 17 00:00:00 2001 From: termie Date: Sat, 12 Feb 2011 13:30:31 -0800 Subject: [PATCH 02/15] fix :returns: and add pep-0257 --- HACKING | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/HACKING b/HACKING index 4e0dfc83..e58d60e5 100644 --- a/HACKING +++ b/HACKING @@ -57,8 +57,12 @@ Docstrings parameters and return types use the Sphinx format. After that an extra newline then close the quotations. + When writing the docstring for a class, an extra line should be placed + after the closing quotations. For more in-depth explanations for these + decisions see http://www.python.org/dev/peps/pep-0257/ + :param foo: the foo parameter :param bar: the bar parameter - :rtype: the return type + :returns: description of the return value """ From 80ac88366c6fc3ecd7436927617120e1080e2a3d Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 14 Feb 2011 10:43:22 -0800 Subject: [PATCH 03/15] fix for bug #716847 - if a volume has not been assigned to a host, then delete from db and skip rpc --- bin/nova-manage | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/nova-manage b/bin/nova-manage index 7835ca55..e4c0684c 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -579,6 +579,13 @@ class VolumeCommands(object): ctxt = context.get_admin_context() volume = db.volume_get(ctxt, param2id(volume_id)) host = volume['host'] + + if not host: + print "Volume not yet assigned to host." + print "Deleting volume from database and skipping rpc." + db.volume_destroy(ctxt, param2id(volume_id)) + return + if volume['status'] == 'in-use': print "Volume is in-use." print "Detach volume from instance and then try again." From 696b52b7bb8906e265d1c30f97f87a8e40ca8f01 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 14 Feb 2011 23:19:15 +0100 Subject: [PATCH 04/15] Resurrect logdir option. --- nova/flags.py | 2 ++ nova/log.py | 11 +++++++++-- nova/twistd.py | 2 -- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 3ba3fe6f..f64a62da 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -282,6 +282,8 @@ DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'), "Top-level directory for maintaining nova's state") +DEFINE_string('logdir', None, 'output to a per-service log file in named ' + 'directory') DEFINE_string('sql_connection', 'sqlite:///$state_path/nova.sqlite', diff --git a/nova/log.py b/nova/log.py index b541488b..a5b4828d 100644 --- a/nova/log.py +++ b/nova/log.py @@ -28,9 +28,11 @@ It also allows setting of formatting information through flags. import cStringIO +import inspect import json import logging import logging.handlers +import os import sys import traceback @@ -123,8 +125,13 @@ def basicConfig(): syslog = SysLogHandler(address='/dev/log') syslog.setFormatter(_formatter) logging.root.addHandler(syslog) - if FLAGS.logfile: - logfile = FileHandler(FLAGS.logfile) + if FLAGS.logfile or FLAGS.logdir: + if FLAGS.logfile: + logfile = FLAGS.logfile + else: + binary = os.path.basename(inspect.stack()[-1][1]) + logpath = '%s.log' % (os.path.join(FLAGS.logdir, binary),) + logfile = FileHandler(logpath) logfile.setFormatter(_formatter) logging.root.addHandler(logfile) diff --git a/nova/twistd.py b/nova/twistd.py index 6390a814..60ff7879 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -43,8 +43,6 @@ else: FLAGS = flags.FLAGS -flags.DEFINE_string('logdir', None, 'directory to keep log files in ' - '(will be prepended to $logfile)') class TwistdServerOptions(ServerOptions): From ff583d8c6f01d137dbcf2128f38a09f41adc4a78 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 14 Feb 2011 23:27:31 +0100 Subject: [PATCH 05/15] Use RotatingFileHandler instead of FileHandler. --- nova/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/log.py b/nova/log.py index b541488b..413dd946 100644 --- a/nova/log.py +++ b/nova/log.py @@ -92,7 +92,7 @@ critical = logging.critical log = logging.log # handlers StreamHandler = logging.StreamHandler -FileHandler = logging.FileHandler +FileHandler = logging.RotatingFileHandler # logging.SysLogHandler is nicer than logging.logging.handler.SysLogHandler. SysLogHandler = logging.handlers.SysLogHandler From ab8b8fea29bb64ac86748b63b0a143fc25e7a660 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 15 Feb 2011 14:02:19 +0100 Subject: [PATCH 06/15] =?UTF-8?q?Na=C3=AFve=20attempt=20at=20threading=20r?= =?UTF-8?q?pc=20requests.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nova/rpc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nova/rpc.py b/nova/rpc.py index 2b1f7298..efe6164a 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -29,6 +29,7 @@ import uuid from carrot import connection as carrot_connection from carrot import messaging +from eventlet import greenpool from eventlet import greenthread from nova import context @@ -155,11 +156,15 @@ class AdapterConsumer(TopicConsumer): def __init__(self, connection=None, topic="broadcast", proxy=None): LOG.debug(_('Initing the Adapter Consumer for %s') % topic) self.proxy = proxy + self.pool = greenpool.GreenPool(1024) super(AdapterConsumer, self).__init__(connection=connection, topic=topic) + def receive(self, *args, **kwargs): + self.pool.spawn_n(self._receive, *args, **kwargs) + @exception.wrap_exception - def receive(self, message_data, message): + def _receive(self, message_data, message): """Magically looks for a method on the proxy object and calls it Message data should be a dictionary with two keys: From cebb4f8bd097ab2945955f171b5dcc39663f75f4 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 15 Feb 2011 22:13:08 +0100 Subject: [PATCH 07/15] Don't hid RotatingFileHandler behind FileHandler's name. --- nova/log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/log.py b/nova/log.py index 413dd946..82a13f6a 100644 --- a/nova/log.py +++ b/nova/log.py @@ -92,7 +92,7 @@ critical = logging.critical log = logging.log # handlers StreamHandler = logging.StreamHandler -FileHandler = logging.RotatingFileHandler +RotatingFileHandler = logging.RotatingFileHandler # logging.SysLogHandler is nicer than logging.logging.handler.SysLogHandler. SysLogHandler = logging.handlers.SysLogHandler @@ -124,7 +124,7 @@ def basicConfig(): syslog.setFormatter(_formatter) logging.root.addHandler(syslog) if FLAGS.logfile: - logfile = FileHandler(FLAGS.logfile) + logfile = RotatingFileHandler(FLAGS.logfile) logfile.setFormatter(_formatter) logging.root.addHandler(logfile) From 546700aa095ca9ade649d3fea9f92287dd041e10 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 15 Feb 2011 23:11:51 +0100 Subject: [PATCH 08/15] Refactor code that decides which logfile to use, if any. Adds unit tests. --- nova/log.py | 17 +++++++++++------ nova/tests/test_log.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/nova/log.py b/nova/log.py index a5b4828d..ec6681ed 100644 --- a/nova/log.py +++ b/nova/log.py @@ -112,6 +112,15 @@ def _dictify_context(context): context = context.to_dict() return context +def _get_binary_name(): + return os.path.basename(inspect.stack()[-1][1]) + +def get_log_file_path(binary=None): + if FLAGS.logfile: + return FLAGS.logfile + if FLAGS.logdir: + binary = binary or _get_binary_name() + return '%s.log' % (os.path.join(FLAGS.logdir, binary),) def basicConfig(): logging.basicConfig() @@ -125,12 +134,8 @@ def basicConfig(): syslog = SysLogHandler(address='/dev/log') syslog.setFormatter(_formatter) logging.root.addHandler(syslog) - if FLAGS.logfile or FLAGS.logdir: - if FLAGS.logfile: - logfile = FLAGS.logfile - else: - binary = os.path.basename(inspect.stack()[-1][1]) - logpath = '%s.log' % (os.path.join(FLAGS.logdir, binary),) + logpath = get_log_file_path() + if logpath: logfile = FileHandler(logpath) logfile.setFormatter(_formatter) logging.root.addHandler(logfile) diff --git a/nova/tests/test_log.py b/nova/tests/test_log.py index 868a5ead..7a5c5293 100644 --- a/nova/tests/test_log.py +++ b/nova/tests/test_log.py @@ -46,6 +46,27 @@ class RootLoggerTestCase(test.TestCase): self.assert_(True) # didn't raise exception +class LogHandlerTestCase(test.TestCase): + def test_log_path_logdir(self): + self.flags(logdir='/some/path') + self.assertEquals(log.get_log_file_path(binary='foo-bar'), + '/some/path/foo-bar.log') + + def test_log_path_logfile(self): + self.flags(logfile='/some/path/foo-bar.log') + self.assertEquals(log.get_log_file_path(binary='foo-bar'), + '/some/path/foo-bar.log') + + def test_log_path_none(self): + self.assertIsNone(log.get_log_file_path(binary='foo-bar')) + + def test_log_path_logfile_overrides_logdir(self): + self.flags(logdir='/some/other/path', + logfile='/some/path/foo-bar.log') + self.assertEquals(log.get_log_file_path(binary='foo-bar'), + '/some/path/foo-bar.log') + + class NovaFormatterTestCase(test.TestCase): def setUp(self): super(NovaFormatterTestCase, self).setUp() From a73b189d540ece0f3ef3653b21ecf9054ad581ac Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 15 Feb 2011 23:13:25 +0100 Subject: [PATCH 09/15] Don't hide RotatingFileHandler behind FileHandler's name. --- nova/log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/log.py b/nova/log.py index 413dd946..32b6d462 100644 --- a/nova/log.py +++ b/nova/log.py @@ -92,7 +92,7 @@ critical = logging.critical log = logging.log # handlers StreamHandler = logging.StreamHandler -FileHandler = logging.RotatingFileHandler +RotatingFileHandler = logging.handlers.RotatingFileHandler # logging.SysLogHandler is nicer than logging.logging.handler.SysLogHandler. SysLogHandler = logging.handlers.SysLogHandler @@ -124,7 +124,7 @@ def basicConfig(): syslog.setFormatter(_formatter) logging.root.addHandler(syslog) if FLAGS.logfile: - logfile = FileHandler(FLAGS.logfile) + logfile = RotatingFileHandler(FLAGS.logfile) logfile.setFormatter(_formatter) logging.root.addHandler(logfile) From acb5e01e2fcf95d2d08ce84cdc60e6f78ed3a033 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 16 Feb 2011 00:10:17 +0100 Subject: [PATCH 10/15] Make rpc thread pool size configurable. --- nova/rpc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/rpc.py b/nova/rpc.py index efe6164a..067d954d 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -43,6 +43,8 @@ from nova import utils FLAGS = flags.FLAGS LOG = logging.getLogger('nova.rpc') +FLAGS.DEFINE_integer('rpc_thread_pool_size', 1024, 'Size of RPC thread pool') + class Connection(carrot_connection.BrokerConnection): """Connection instance object""" @@ -156,7 +158,7 @@ class AdapterConsumer(TopicConsumer): def __init__(self, connection=None, topic="broadcast", proxy=None): LOG.debug(_('Initing the Adapter Consumer for %s') % topic) self.proxy = proxy - self.pool = greenpool.GreenPool(1024) + self.pool = greenpool.GreenPool(FLAGS.rpc_thread_pool_size) super(AdapterConsumer, self).__init__(connection=connection, topic=topic) From f6d671f6ce0ee413eec538e1ae70801cad74276e Mon Sep 17 00:00:00 2001 From: Vasiliy Shlykov Date: Wed, 16 Feb 2011 10:13:52 +0300 Subject: [PATCH 11/15] Added myself to the authors file. --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index b359fec2..c338a9b3 100644 --- a/Authors +++ b/Authors @@ -56,6 +56,7 @@ Thierry Carrez Todd Willey Trey Morris Tushar Patil +Vasiliy Shlykov Vishvananda Ishaya Youcef Laribi Zhixue Wu From 4cf7fe2cc6f8bc57afbd4edcf5d63d3bf8817747 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 16 Feb 2011 10:02:17 +0100 Subject: [PATCH 12/15] assertIsNone is a 2.7-ism. --- nova/tests/test_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/test_log.py b/nova/tests/test_log.py index 7a5c5293..c2c9d777 100644 --- a/nova/tests/test_log.py +++ b/nova/tests/test_log.py @@ -58,7 +58,7 @@ class LogHandlerTestCase(test.TestCase): '/some/path/foo-bar.log') def test_log_path_none(self): - self.assertIsNone(log.get_log_file_path(binary='foo-bar')) + self.assertTrue(log.get_log_file_path(binary='foo-bar') is None) def test_log_path_logfile_overrides_logdir(self): self.flags(logdir='/some/other/path', From 5280e42a5b1904ceeba3738184f62caab489181e Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 16 Feb 2011 10:46:59 +0100 Subject: [PATCH 13/15] Fix PEP-8 stuff --- nova/log.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/log.py b/nova/log.py index ec6681ed..90c40cca 100644 --- a/nova/log.py +++ b/nova/log.py @@ -112,9 +112,11 @@ def _dictify_context(context): context = context.to_dict() return context + def _get_binary_name(): return os.path.basename(inspect.stack()[-1][1]) + def get_log_file_path(binary=None): if FLAGS.logfile: return FLAGS.logfile @@ -122,6 +124,7 @@ def get_log_file_path(binary=None): binary = binary or _get_binary_name() return '%s.log' % (os.path.join(FLAGS.logdir, binary),) + def basicConfig(): logging.basicConfig() for handler in logging.root.handlers: From 170d3d5ab3023ac99a542fe19298f2783d423060 Mon Sep 17 00:00:00 2001 From: Brian Schott Date: Wed, 16 Feb 2011 13:24:46 -0500 Subject: [PATCH 14/15] fixed authors, import sys in migration.py --- Authors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Authors b/Authors index 0edabc53..29fbf76b 100644 --- a/Authors +++ b/Authors @@ -4,7 +4,7 @@ Anthony Young Antony Messerli Armando Migliaccio Bilal Akhtar -Brian Schott +Brian Schott Chiradeep Vittal Chmouel Boudjnah Chris Behrens From fdfc2ee12c60aa4cc056ccf7267b1ab130ea4ca1 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 16 Feb 2011 20:34:17 +0100 Subject: [PATCH 15/15] Spell flags correctly (i.e. not in upper case) --- nova/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/rpc.py b/nova/rpc.py index 067d954d..205bb524 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -43,7 +43,7 @@ from nova import utils FLAGS = flags.FLAGS LOG = logging.getLogger('nova.rpc') -FLAGS.DEFINE_integer('rpc_thread_pool_size', 1024, 'Size of RPC thread pool') +flags.DEFINE_integer('rpc_thread_pool_size', 1024, 'Size of RPC thread pool') class Connection(carrot_connection.BrokerConnection):