From 9ff79304ec1f95cc501529358f2155d78f7b654d Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 27 Jun 2011 20:21:45 -0700 Subject: [PATCH 01/15] getting started --- nova/rpc.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nova/rpc.py b/nova/rpc.py index 2e78a31e7..d4ac19c20 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -45,6 +45,8 @@ from nova import flags from nova import log as logging from nova import utils +from nova.notifier import api as notifier + LOG = logging.getLogger('nova.rpc') @@ -312,6 +314,7 @@ class ConsumerSet(object): if not it: break while True: + ex = None try: it.next() except StopIteration: @@ -319,7 +322,17 @@ class ConsumerSet(object): except greenlet.GreenletExit: running = False break + except exception.NovaException, e: + if not e.notification_level: + ex = e + # We have an exception we can + # tell the Notification system about. + # Pass it on. + except Exception as e: + ex = e + + if ex: LOG.exception(_("Exception while processing consumer")) self.reconnect() # Break to outer loop From 4f625513acc4eef03c4fff12dbb3b772546853a8 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 28 Jun 2011 05:51:40 -0700 Subject: [PATCH 02/15] refactoring to compute from scheduler --- nova/notifier/api.py | 39 +++++++++++++++++++++++++++++++++++++++ nova/rpc.py | 9 ++------- nova/scheduler/driver.py | 10 ++++++++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/nova/notifier/api.py b/nova/notifier/api.py index d49517c8b..027aa7cc3 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -37,6 +37,45 @@ class BadPriorityException(Exception): pass +def publisher_id(service, host=None): + if not host: + host = FLAGS.host + return "%s.%s" % (service, host) + + +def msgkeys(event_type, instance_id, level, publisher_id): + return dict(event_type=event_type, instance_id=instance_id, + notification_level=level, publisher_id=publisher_id) + + +def safe_notify(publisher_id, event_type, priority, payload): + try: + notify(publisher_id, event_type, notification_level, payload) + exception Exception, e: + LOG.exception(_("Problem '%(e)' attempting to " + "send to notification system." % locals())) + + +def instance_safe_notify(publisher_id, event_type, priority, instance_id, + extra_payload=None): + payload = dict(instance_id = instance_id) + if extra_payload: + payload.extend(extra_payload) + safe_notify(publisher_id, event_type, priority, payload) + + +def exception_to_notification(self, ex): + required = ['instance_id', 'publisher_id', 'notification_level', + 'event_type'] + for key in required: + if not (hasattr(ex, key) and ex.key): + return # Doesn't have everything we need. Skip it. + instance_id = ex.instance_id + publisher_id = ex.publisher_id + notification_level = ex.notification_level + event_type = ex.event_type + instance_safe_notify(publisher_id, event_type, priority, instance_id) + def notify(publisher_id, event_type, priority, payload): """ Sends a notification using the specified driver diff --git a/nova/rpc.py b/nova/rpc.py index d4ac19c20..47d63769a 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -44,7 +44,6 @@ from nova import fakerabbit from nova import flags from nova import log as logging from nova import utils - from nova.notifier import api as notifier @@ -323,12 +322,8 @@ class ConsumerSet(object): running = False break except exception.NovaException, e: - if not e.notification_level: - ex = e - # We have an exception we can - # tell the Notification system about. - # Pass it on. - + ex = e + notifier.exception_to_notification(e) except Exception as e: ex = e diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 0b257c5d8..7e5a15c7f 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -27,6 +27,7 @@ from nova import db from nova import exception from nova import flags from nova import log as logging +from nova import notifier from nova import rpc from nova import utils from nova.compute import power_state @@ -47,6 +48,15 @@ class WillNotSchedule(exception.Error): pass +def publisher_id(host=None): + return notifier.publisher_id("scheduler", host) + + +def notifier(instance_id, event_type, level, host=None): + return dict(instance_id=instance_id, event_type=event_type, + notification_level=level, host=publisher_id(host)) + + class Scheduler(object): """The base class that all Scheduler clases should inherit from.""" From 898f4c46f89b0675b74a618783bde52a842a8fcc Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 28 Jun 2011 21:04:50 -0700 Subject: [PATCH 03/15] moved to wrap_exception decorator --- nova/exception.py | 8 +++++++- nova/notifier/api.py | 26 -------------------------- nova/rpc.py | 8 -------- 3 files changed, 7 insertions(+), 35 deletions(-) diff --git a/nova/exception.py b/nova/exception.py index f3a452228..4b625dd04 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -81,11 +81,17 @@ def wrap_db_error(f): _wrap.func_name = f.func_name -def wrap_exception(f): +def wrap_exception(f, notifier=None, publisher_id=None, level=None): def _wrap(*args, **kw): try: return f(*args, **kw) except Exception, e: + if notifier != None and 'safe_notify' in notifier.dir(): + event_type = f.__name__ + payload = dict(args=args, exception=e) + payload.update(kw) + notifier.safe_notify(publisher_id, event_type, level, payload) + if not isinstance(e, Error): #exc_type, exc_value, exc_traceback = sys.exc_info() LOG.exception(_('Uncaught exception')) diff --git a/nova/notifier/api.py b/nova/notifier/api.py index 027aa7cc3..89527be16 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -43,11 +43,6 @@ def publisher_id(service, host=None): return "%s.%s" % (service, host) -def msgkeys(event_type, instance_id, level, publisher_id): - return dict(event_type=event_type, instance_id=instance_id, - notification_level=level, publisher_id=publisher_id) - - def safe_notify(publisher_id, event_type, priority, payload): try: notify(publisher_id, event_type, notification_level, payload) @@ -55,27 +50,6 @@ def safe_notify(publisher_id, event_type, priority, payload): LOG.exception(_("Problem '%(e)' attempting to " "send to notification system." % locals())) - -def instance_safe_notify(publisher_id, event_type, priority, instance_id, - extra_payload=None): - payload = dict(instance_id = instance_id) - if extra_payload: - payload.extend(extra_payload) - safe_notify(publisher_id, event_type, priority, payload) - - -def exception_to_notification(self, ex): - required = ['instance_id', 'publisher_id', 'notification_level', - 'event_type'] - for key in required: - if not (hasattr(ex, key) and ex.key): - return # Doesn't have everything we need. Skip it. - instance_id = ex.instance_id - publisher_id = ex.publisher_id - notification_level = ex.notification_level - event_type = ex.event_type - instance_safe_notify(publisher_id, event_type, priority, instance_id) - def notify(publisher_id, event_type, priority, payload): """ Sends a notification using the specified driver diff --git a/nova/rpc.py b/nova/rpc.py index 47d63769a..2e78a31e7 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -44,7 +44,6 @@ from nova import fakerabbit from nova import flags from nova import log as logging from nova import utils -from nova.notifier import api as notifier LOG = logging.getLogger('nova.rpc') @@ -313,7 +312,6 @@ class ConsumerSet(object): if not it: break while True: - ex = None try: it.next() except StopIteration: @@ -321,13 +319,7 @@ class ConsumerSet(object): except greenlet.GreenletExit: running = False break - except exception.NovaException, e: - ex = e - notifier.exception_to_notification(e) except Exception as e: - ex = e - - if ex: LOG.exception(_("Exception while processing consumer")) self.reconnect() # Break to outer loop From 1f1f8387a649785860883133235138182286f18e Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 29 Jun 2011 08:14:43 -0700 Subject: [PATCH 04/15] moved to wrap_exception approach --- nova/exception.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/exception.py b/nova/exception.py index 4b625dd04..fb2094c04 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -25,6 +25,7 @@ SHOULD include dedicated exception logging. """ from nova import log as logging +from nova.notifier import api as notifier LOG = logging.getLogger('nova.exception') @@ -81,13 +82,12 @@ def wrap_db_error(f): _wrap.func_name = f.func_name -def wrap_exception(f, notifier=None, publisher_id=None, level=None): +def wrap_exception(f, event_type=None, publisher_id=None, level=notifier.ERROR): def _wrap(*args, **kw): try: return f(*args, **kw) except Exception, e: - if notifier != None and 'safe_notify' in notifier.dir(): - event_type = f.__name__ + if event_type and publisher_id: payload = dict(args=args, exception=e) payload.update(kw) notifier.safe_notify(publisher_id, event_type, level, payload) From 57e3187cd990fdad2d6bacdd0d442ada6d52adf9 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 29 Jun 2011 19:10:11 -0700 Subject: [PATCH 05/15] done and done --- nova/exception.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/exception.py b/nova/exception.py index c6d2bbc3d..ea590199c 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -82,7 +82,8 @@ def wrap_db_error(f): _wrap.func_name = f.func_name -def wrap_exception(notifier=None, publisher_id=None, event_type=None, level=None): +def wrap_exception(notifier=None, publisher_id=None, event_type=None, + level=None): """This decorator wraps a method to catch any exceptions that may get thrown. It logs the exception as well as optionally sending it to the notification system. From b7e6d25c53fec02e26ac631ae0bdb413a13fb858 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 29 Jun 2011 20:02:30 -0700 Subject: [PATCH 06/15] clean up --- nova/scheduler/driver.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index db392a607..ec2c9ef78 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -30,7 +30,6 @@ from nova import log as logging from nova import rpc from nova import utils from nova.compute import power_state -from nova.notifier import api as notifier_api FLAGS = flags.FLAGS @@ -49,10 +48,6 @@ class WillNotSchedule(exception.Error): pass -def publisher_id(host=None): - return notifier_api.publisher_id("scheduler", host) - - class Scheduler(object): """The base class that all Scheduler clases should inherit from.""" From 6db4a23a8f65c4ff063806e1dd9fd9830f3c9a22 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 1 Jul 2011 07:31:17 -0700 Subject: [PATCH 07/15] review fixes --- nova/exception.py | 4 ++-- nova/notifier/api.py | 14 +++++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/nova/exception.py b/nova/exception.py index ea590199c..6e277b68d 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -113,8 +113,8 @@ def wrap_exception(notifier=None, publisher_id=None, event_type=None, # propagated. temp_type = f.__name__ - notifier.safe_notify(publisher_id, temp_type, temp_level, - payload) + notifier.notify(publisher_id, temp_type, temp_level, + payload) if not isinstance(e, Error): #exc_type, exc_value, exc_traceback = sys.exc_info() diff --git a/nova/notifier/api.py b/nova/notifier/api.py index d388eda96..98969fd3e 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -45,14 +45,6 @@ def publisher_id(service, host=None): return "%s.%s" % (service, host) -def safe_notify(publisher_id, event_type, priority, payload): - try: - notify(publisher_id, event_type, notification_level, payload) - except Exception, e: - LOG.exception(_("Problem '%(e)s' attempting to " - "send to notification system." % locals())) - - def notify(publisher_id, event_type, priority, payload): """ Sends a notification using the specified driver @@ -95,4 +87,8 @@ def notify(publisher_id, event_type, priority, payload): priority=priority, payload=payload, timestamp=str(utils.utcnow())) - driver.notify(msg) + try: + driver.notify(msg) + except Exception, e: + LOG.exception(_("Problem '%(e)s' attempting to " + "send to notification system." % locals())) From 21c906fcfbb78760958d150944b804b47bd4499a Mon Sep 17 00:00:00 2001 From: Mike Scherbakov Date: Sat, 2 Jul 2011 01:28:13 +0400 Subject: [PATCH 08/15] Improvements to nova-manage: network list now includes vlan and projectID, added servers list filtered by zone if needed --- bin/nova-manage | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 7dfe91698..8ba78a56a 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -615,15 +615,19 @@ class NetworkCommands(object): def list(self): """List all created networks""" - print "%-18s\t%-15s\t%-15s\t%-15s" % (_('network'), - _('netmask'), - _('start address'), - 'DNS') + print "%-18s\t%-15s\t%-15s\t%-15s\t%-15s\t%-15s" % (_('network'), + _('netmask'), + _('start address'), + _('DNS'), + _('VlanID'), + 'project') for network in db.network_get_all(context.get_admin_context()): - print "%-18s\t%-15s\t%-15s\t%-15s" % (network.cidr, - network.netmask, - network.dhcp_start, - network.dns) + print "%-18s\t%-15s\t%-15s\t%-15s\t%-15s\t%-15s" % (network.cidr, + network.netmask, + network.dhcp_start, + network.dns, + network.vlan, + network.project_id) def delete(self, fixed_range): """Deletes a network""" @@ -812,6 +816,28 @@ class ServiceCommands(object): {"method": "update_available_resource"}) +class ServerCommands(object): + """List servers""" + + def list(self, zone=None): + """Show a list of all servers. Filter by zone. + args: [zone]""" + print "%-25s\t%-15s" % (_('host'), + _('zone')) + ctxt = context.get_admin_context() + now = utils.utcnow() + services = db.service_get_all(ctxt) + if zone: + services = [s for s in services if s['availability_zone'] == zone] + servers = [] + for srv in services: + if not [s for s in servers if s['host'] == srv['host']]: + servers.append(srv) + + for srv in servers: + print "%-25s\t%-15s" % (srv['host'], srv['availability_zone']) + + class DbCommands(object): """Class for managing the database.""" @@ -1191,6 +1217,7 @@ CATEGORIES = [ ('project', ProjectCommands), ('role', RoleCommands), ('service', ServiceCommands), + ('server', ServerCommands), ('shell', ShellCommands), ('user', UserCommands), ('version', VersionCommands), From 587b73c3a831db987be892220d38cf43e9f4150c Mon Sep 17 00:00:00 2001 From: Mike Scherbakov Date: Tue, 12 Jul 2011 11:07:30 -0700 Subject: [PATCH 09/15] Renamed 'nova-manage server list' -> 'nova-manage host list' to differentiate physical hosts from VMs --- bin/nova-manage | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 8ba78a56a..94ab22092 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -816,11 +816,11 @@ class ServiceCommands(object): {"method": "update_available_resource"}) -class ServerCommands(object): - """List servers""" +class HostCommands(object): + """List hosts""" def list(self, zone=None): - """Show a list of all servers. Filter by zone. + """Show a list of all physical hosts. Filter by zone. args: [zone]""" print "%-25s\t%-15s" % (_('host'), _('zone')) @@ -829,13 +829,13 @@ class ServerCommands(object): services = db.service_get_all(ctxt) if zone: services = [s for s in services if s['availability_zone'] == zone] - servers = [] + hosts = [] for srv in services: - if not [s for s in servers if s['host'] == srv['host']]: - servers.append(srv) + if not [h for h in hosts if h['host'] == srv['host']]: + hosts.append(srv) - for srv in servers: - print "%-25s\t%-15s" % (srv['host'], srv['availability_zone']) + for h in hosts: + print "%-25s\t%-15s" % (h['host'], h['availability_zone']) class DbCommands(object): @@ -1211,13 +1211,13 @@ CATEGORIES = [ ('fixed', FixedIpCommands), ('flavor', InstanceTypeCommands), ('floating', FloatingIpCommands), + ('host', HostCommands), ('instance_type', InstanceTypeCommands), ('image', ImageCommands), ('network', NetworkCommands), ('project', ProjectCommands), ('role', RoleCommands), ('service', ServiceCommands), - ('server', ServerCommands), ('shell', ShellCommands), ('user', UserCommands), ('version', VersionCommands), From 6f445dda3ec07afd38a1b1a4a0761df69a28679d Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 13 Jul 2011 08:34:41 -0400 Subject: [PATCH 10/15] Added Mohammed Naser to Authors file. --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 4aa65eea2..2e50cfbe0 100644 --- a/Authors +++ b/Authors @@ -65,6 +65,7 @@ Masanori Itoh Matt Dietz Michael Gundlach Mike Scherbakov +Mohammed Naser Monsyne Dragon Monty Taylor MORITA Kazutaka From f16b77e2c2bd8f122c94b136fe6e07a9d31514d5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 13 Jul 2011 17:16:27 -0400 Subject: [PATCH 11/15] support '-' to indicate stdout in nova-manage project 'environment' and 'zip' This just adds support to do: nova-manage project zip test-project admin - > out.zip nova-manage project environment test-project admin - | grep NOVA_URL --- bin/nova-manage | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 7dfe91698..b5247f8c3 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -414,8 +414,11 @@ class ProjectCommands(object): except (exception.UserNotFound, exception.ProjectNotFound) as ex: print ex raise - with open(filename, 'w') as f: - f.write(rc) + if filename == "-": + f = sys.stdout + else: + f = open(filename, 'w') + f.write(rc) def list(self, username=None): """Lists all projects @@ -465,8 +468,11 @@ class ProjectCommands(object): arguments: project_id user_id [filename='nova.zip]""" try: zip_file = self.manager.get_credentials(user_id, project_id) - with open(filename, 'w') as f: - f.write(zip_file) + if filename == "-": + f = sys.stdout + else: + f = open(filename, 'w') + f.write(zip_file) except (exception.UserNotFound, exception.ProjectNotFound) as ex: print ex raise From 97b072a9d5694a2084eeb621f99591111ec554ff Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 13 Jul 2011 22:01:52 -0400 Subject: [PATCH 12/15] use 'with' so that close is called on file handle --- bin/nova-manage | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index b5247f8c3..a83fe70f8 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -415,10 +415,10 @@ class ProjectCommands(object): print ex raise if filename == "-": - f = sys.stdout + sys.stdout.write(rc) else: - f = open(filename, 'w') - f.write(rc) + with open(filename, 'w') as f: + f.write(rc) def list(self, username=None): """Lists all projects @@ -469,10 +469,10 @@ class ProjectCommands(object): try: zip_file = self.manager.get_credentials(user_id, project_id) if filename == "-": - f = sys.stdout + sys.stdout.write(zip_file) else: - f = open(filename, 'w') - f.write(zip_file) + with open(filename, 'w') as f: + f.write(zip_file) except (exception.UserNotFound, exception.ProjectNotFound) as ex: print ex raise From 97965be0e0e93ba9ca75c3f6397c3f27078c4cf3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 14 Jul 2011 07:08:02 -0400 Subject: [PATCH 13/15] add self to authors --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 2e50cfbe0..a366cec87 100644 --- a/Authors +++ b/Authors @@ -85,6 +85,7 @@ Ryan Lucio Salvatore Orlando Sandy Walsh Sateesh Chodapuneedi +Scott Moser Soren Hansen Thierry Carrez Todd Willey From 1aec00ef0c9984a503d439788823723f18aa2b82 Mon Sep 17 00:00:00 2001 From: Stephanie Reese Date: Thu, 14 Jul 2011 23:12:42 -0400 Subject: [PATCH 15/15] Updated Authors --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 2e50cfbe0..62edd6771 100644 --- a/Authors +++ b/Authors @@ -86,6 +86,7 @@ Salvatore Orlando Sandy Walsh Sateesh Chodapuneedi Soren Hansen +Stephanie Reese Thierry Carrez Todd Willey Trey Morris