Import openstack.common.log and use inside heat infrastructure

Allows heat to match openstack-common log calling conventions such
as using a context object or instance UUID for better traceability.

Change-Id: Idaa6c04270d9d7143c85988d685c0f9e241b635b
Signed-off-by: Steven Dake <sdake@redhat.com>

bin/heat-api

@@ -33,12 +33,13 @@ if os.path.exists(os.path.join(possible_topdir, 'heat', '__init__.py')):
 
 gettext.install('heat', unicode=1)
 
-import logging
 from heat import rpc
 from heat.common import config
 from heat.common import wsgi
 from paste import httpserver
 
+from heat.openstack.common import log as logging
+
 LOG = logging.getLogger('heat.api')
 
 if __name__ == '__main__':

bin/heat-engine

@@ -26,7 +26,8 @@ eventlet.monkey_patch()
 
 import os
 import sys
-import logging
+
+from heat.openstack.common import log as logging
 
 # If ../heat/__init__.py exists, add ../ to Python search path, so that
 # it will override what happens to be installed in /usr/(local/)lib/python...

bin/heat-metadata

@@ -33,13 +33,14 @@ if os.path.exists(os.path.join(possible_topdir, 'heat', '__init__.py')):
 
 gettext.install('heat', unicode=1)
 
-import logging
 from heat import rpc
 from heat.common import config
 from heat.common import wsgi
 from heat.common import context
 from paste import httpserver
 
+from heat.openstack.common import log as logging
+
 LOG = logging.getLogger('heat.metadata')
 
 

heat/api/middleware/version_negotiation.py

@@ -19,9 +19,10 @@ and/or Accept headers and attempts to negotiate an API controller to
 return
 """
 
-import logging
 import re
 
+from heat.openstack.common import log as logging
+
 from heat.api import versions
 from heat.common import wsgi
 

heat/api/v1/__init__.py

@@ -16,7 +16,6 @@
 import json
 import urlparse
 import httplib
-import logging
 import routes
 import gettext
 
@@ -31,6 +30,8 @@ from heat import utils
 from heat.common import context
 from heat.api.v1 import exception
 
+from heat.openstack.common import log as logging
+
 logger = logging.getLogger(__name__)
 
 

heat/api/v1/stacks.py

@@ -18,7 +18,6 @@
 """
 import httplib
 import json
-import logging
 import os
 import socket
 import sys
@@ -33,6 +32,7 @@ from heat import utils
 from heat import rpc
 import heat.rpc.common as rpc_common
 
+from heat.openstack.common import log as logging
 
 logger = logging.getLogger('heat.api.v1.stacks')
 

heat/client.py

@@ -18,11 +18,13 @@ Client classes for callers of a heat system
 """
 
 from lxml import etree
-import logging
 import os
 from heat.common import client as base_client
 from heat.common import exception
 from heat.cloudformations import *
+
+from heat.openstack.common import log as logging
+
 logger = logging.getLogger(__name__)
 
 

heat/db/sqlalchemy/session.py

@@ -14,11 +14,12 @@
 
 """Session Handling for SQLAlchemy backend."""
 
-import logging
 import sqlalchemy.interfaces
 import sqlalchemy.orm
 from sqlalchemy.exc import DisconnectionError
 
+from heat.openstack.common import log as logging
+
 from heat.db import api as db_api
 from heat.openstack.common import cfg
 

heat/engine/api.py

@@ -13,10 +13,10 @@
 #    under the License.
 
 import re
-import logging
 from heat.common import utils as heat_utils
 from heat.engine import parser
 
+from heat.openstack.common import log as logging
 
 logger = logging.getLogger('heat.engine.manager')
 

heat/engine/auth.py

@@ -13,7 +13,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import logging
 import json
 import httplib
 import urlparse
@@ -22,6 +21,7 @@ from novaclient.exceptions import BadRequest
 from novaclient.exceptions import NotFound
 from novaclient.exceptions import AuthorizationFailure
 from heat.common import context
+from heat.openstack.common import log as logging
 
 logger = logging.getLogger('heat.engine.auth')
 

heat/engine/autoscaling.py

@@ -14,7 +14,6 @@
 #    under the License.
 
 import eventlet
-import logging
 import json
 import os
 
@@ -23,6 +22,8 @@ from heat.db import api as db_api
 from heat.engine import instance
 from heat.engine.resources import Resource
 
+from heat.openstack.common import log as logging
+
 logger = logging.getLogger('heat.engine.autoscaling')
 
 

heat/engine/checkeddict.py

@@ -15,7 +15,8 @@
 
 import collections
 import re
-import logging
+
+from heat.openstack.common import log as logging
 
 logger = logging.getLogger('heat.engine.checkeddict')
 

heat/engine/cloud_watch.py

@@ -14,7 +14,6 @@
 #    under the License.
 
 import eventlet
-import logging
 import json
 import os
 
@@ -22,6 +21,8 @@ from heat.common import exception
 from heat.db import api as db_api
 from heat.engine.resources import Resource
 
+from heat.openstack.common import log as logging
+
 logger = logging.getLogger('heat.engine.cloud_watch')
 
 

heat/engine/eip.py

@@ -13,12 +13,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import logging
-
 from heat.common import exception
 from heat.engine.resources import Resource
 from novaclient.exceptions import NotFound
 
+from heat.openstack.common import log as logging
+
 logger = logging.getLogger('heat.engine.eip')
 
 

heat/engine/instance.py

@@ -14,7 +14,6 @@
 #    under the License.
 
 import eventlet
-import logging
 import os
 import json
 import sys
@@ -26,6 +25,8 @@ import heat
 from heat.engine import resources
 from heat.common import exception
 
+from heat.openstack.common import log as logging
+
 logger = logging.getLogger('heat.engine.instance')
 
 

heat/engine/loadbalancer.py

@@ -15,7 +15,6 @@
 
 import urllib2
 import json
-import logging
 
 from heat.common import exception
 from heat.engine import stack
@@ -23,6 +22,8 @@ from heat.db import api as db_api
 from heat.engine import parser
 from novaclient.exceptions import NotFound
 
+from heat.openstack.common import log as logging
+
 logger = logging.getLogger(__file__)
 
 lb_template = '''

heat/engine/manager.py

@@ -16,7 +16,6 @@
 
 from copy import deepcopy
 import datetime
-import logging
 import webob
 import json
 import urlparse
@@ -33,7 +32,9 @@ from heat.engine import parser
 from heat.engine import resources
 from heat.engine import watchrule
 from heat.engine import auth
+
 from heat.openstack.common import timeutils
+from heat.openstack.common import log as logging
 
 from novaclient.v1_1 import client
 from novaclient.exceptions import BadRequest

heat/engine/parser.py

@@ -17,7 +17,6 @@ import eventlet
 import json
 import functools
 import copy
-import logging
 
 from heat.common import exception
 from heat.engine import checkeddict
@@ -25,6 +24,7 @@ from heat.engine import dependencies
 from heat.engine import resources
 from heat.db import api as db_api
 
+from heat.openstack.common import log as logging
 
 logger = logging.getLogger('heat.engine.parser')
 

heat/engine/resources.py

@@ -15,7 +15,6 @@
 
 import base64
 from datetime import datetime
-import logging
 
 from novaclient.v1_1 import client as nc
 from keystoneclient.v2_0 import client as kc
@@ -26,6 +25,8 @@ from heat.db import api as db_api
 from heat.engine import checkeddict
 from heat.engine import auth
 
+from heat.openstack.common import log as logging
+
 logger = logging.getLogger('heat.engine.resources')
 
 

heat/engine/security_group.py

@@ -13,12 +13,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import logging
 from novaclient.exceptions import BadRequest
 from novaclient.exceptions import NotFound
 from heat.common import exception
 from heat.engine.resources import Resource
 
+from heat.openstack.common import log as logging
+
 logger = logging.getLogger('heat.engine.security_group')
 
 

heat/engine/stack.py

@@ -15,13 +15,14 @@
 
 import urllib2
 import json
-import logging
 
 from heat.common import exception
 from heat.engine.resources import Resource
 from heat.db import api as db_api
 from heat.engine import parser
 
+from heat.openstack.common import log as logging
+
 logger = logging.getLogger(__file__)
 
 

heat/engine/user.py

@@ -14,10 +14,10 @@
 #    under the License.
 
 import eventlet
-import logging
 from heat.common import exception
 from heat.engine.resources import Resource
 
+from heat.openstack.common import log as logging
 
 logger = logging.getLogger('heat.engine.user')
 

heat/engine/volume.py

@@ -14,7 +14,7 @@
 #    under the License.
 
 import eventlet
-import logging
+from heat.openstack.common import log as logging
 import re
 
 from heat.common import exception

heat/engine/wait_condition.py

@@ -14,12 +14,13 @@
 #    under the License.
 
 import eventlet
-import logging
 import json
 
 from heat.common import exception
 from heat.engine import resources
 
+from heat.openstack.common import log as logging
+
 logger = logging.getLogger('heat.engine.wait_condition')
 
 

heat/engine/watchrule.py

@@ -15,7 +15,7 @@
 
 
 import datetime
-import logging
+from heat.openstack.common import log as logging
 from heat.openstack.common import timeutils
 
 logger = logging.getLogger('heat.engine.watchrule')

heat/manager.py

@@ -53,11 +53,12 @@ This module provides Manager, a base class for managers.
 
 """
 
-import logging
-
 from heat import version
 from heat.common import config
 
+from heat.openstack.common import log as logging
+
+
 FLAGS = config.FLAGS
 LOG = logging.getLogger(__name__)
 

heat/metadata/api/v1/__init__.py

@@ -13,12 +13,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import logging
 import routes
 
 from heat.common import wsgi
 from heat.metadata.api.v1 import metadata
 
+from heat.openstack.common import log as logging
+
 
 class API(wsgi.Router):
     """

heat/openstack/common/jsonutils.py

@@ -0,0 +1,144 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 Justin Santa Barbara
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+'''
+JSON related utilities.
+
+This module provides a few things:
+
+    1) A handy function for getting an object down to something that can be
+    JSON serialized.  See to_primitive().
+
+    2) Wrappers around loads() and dumps().  The dumps() wrapper will
+    automatically use to_primitive() for you if needed.
+
+    3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
+    is available.
+'''
+
+
+import datetime
+import inspect
+import itertools
+import json
+import xmlrpclib
+
+
+def to_primitive(value, convert_instances=False, level=0):
+    """Convert a complex object into primitives.
+
+    Handy for JSON serialization. We can optionally handle instances,
+    but since this is a recursive function, we could have cyclical
+    data structures.
+
+    To handle cyclical data structures we could track the actual objects
+    visited in a set, but not all objects are hashable. Instead we just
+    track the depth of the object inspections and don't go too deep.
+
+    Therefore, convert_instances=True is lossy ... be aware.
+
+    """
+    nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod,
+             inspect.isfunction, inspect.isgeneratorfunction,
+             inspect.isgenerator, inspect.istraceback, inspect.isframe,
+             inspect.iscode, inspect.isbuiltin, inspect.isroutine,
+             inspect.isabstract]
+    for test in nasty:
+        if test(value):
+            return unicode(value)
+
+    # value of itertools.count doesn't get caught by inspects
+    # above and results in infinite loop when list(value) is called.
+    if type(value) == itertools.count:
+        return unicode(value)
+
+    # FIXME(vish): Workaround for LP bug 852095. Without this workaround,
+    #              tests that raise an exception in a mocked method that
+    #              has a @wrap_exception with a notifier will fail. If
+    #              we up the dependency to 0.5.4 (when it is released) we
+    #              can remove this workaround.
+    if getattr(value, '__module__', None) == 'mox':
+        return 'mock'
+
+    if level > 3:
+        return '?'
+
+    # The try block may not be necessary after the class check above,
+    # but just in case ...
+    try:
+        # It's not clear why xmlrpclib created their own DateTime type, but
+        # for our purposes, make it a datetime type which is explicitly
+        # handled
+        if isinstance(value, xmlrpclib.DateTime):
+            value = datetime.datetime(*tuple(value.timetuple())[:6])
+
+        if isinstance(value, (list, tuple)):
+            o = []
+            for v in value:
+                o.append(to_primitive(v, convert_instances=convert_instances,
+                                      level=level))
+            return o
+        elif isinstance(value, dict):
+            o = {}
+            for k, v in value.iteritems():
+                o[k] = to_primitive(v, convert_instances=convert_instances,
+                                    level=level)
+            return o
+        elif isinstance(value, datetime.datetime):
+            return str(value)
+        elif hasattr(value, 'iteritems'):
+            return to_primitive(dict(value.iteritems()),
+                                convert_instances=convert_instances,
+                                level=level)
+        elif hasattr(value, '__iter__'):
+            return to_primitive(list(value), level)
+        elif convert_instances and hasattr(value, '__dict__'):
+            # Likely an instance of something. Watch for cycles.
+            # Ignore class member vars.
+            return to_primitive(value.__dict__,
+                                convert_instances=convert_instances,
+                                level=level + 1)
+        else:
+            return value
+    except TypeError, e:
+        # Class objects are tricky since they may define something like
+        # __iter__ defined but it isn't callable as list().
+        return unicode(value)
+
+
+def dumps(value, default=to_primitive, **kwargs):
+    return json.dumps(value, default=default, **kwargs)
+
+
+def loads(s):
+    return json.loads(s)
+
+
+def load(s):
+    return json.load(s)
+
+
+try:
+    import anyjson
+except ImportError:
+    pass
+else:
+    anyjson._modules.append((__name__, 'dumps', TypeError,
+                                       'loads', ValueError, 'load'))
+    anyjson.force_implementation(__name__)

heat/openstack/common/log.py

@@ -0,0 +1,459 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Openstack logging handler.
+
+This module adds to logging functionality by adding the option to specify
+a context object when calling the various log methods.  If the context object
+is not specified, default formatting is used. Additionally, an instance uuid
+may be passed as part of the log message, which is intended to make it easier
+for admins to find messages related to a specific instance.
+
+It also allows setting of formatting information through conf.
+
+"""
+
+import cStringIO
+import inspect
+import itertools
+import logging
+import logging.config
+import logging.handlers
+import os
+import stat
+import sys
+import traceback
+
+from heat.openstack.common import cfg
+from heat.openstack.common.gettextutils import _
+from heat.openstack.common import jsonutils
+from heat.openstack.common import local
+from heat.openstack.common import notifier
+
+
+log_opts = [
+    cfg.StrOpt('logging_context_format_string',
+               default='%(asctime)s %(levelname)s %(name)s [%(request_id)s '
+                       '%(user_id)s %(project_id)s] %(instance)s'
+                       '%(message)s',
+               help='format string to use for log messages with context'),
+    cfg.StrOpt('logging_default_format_string',
+               default='%(asctime)s %(levelname)s %(name)s [-] %(instance)s'
+                       '%(message)s',
+               help='format string to use for log messages without context'),
+    cfg.StrOpt('logging_debug_format_suffix',
+               default='from (pid=%(process)d) %(funcName)s '
+                       '%(pathname)s:%(lineno)d',
+               help='data to append to log format when level is DEBUG'),
+    cfg.StrOpt('logging_exception_prefix',
+               default='%(asctime)s TRACE %(name)s %(instance)s',
+               help='prefix each line of exception output with this format'),
+    cfg.ListOpt('default_log_levels',
+                default=[
+                  'amqplib=WARN',
+                  'sqlalchemy=WARN',
+                  'boto=WARN',
+                  'suds=INFO',
+                  'keystone=INFO',
+                  'eventlet.wsgi.server=WARN'
+                  ],
+                help='list of logger=LEVEL pairs'),
+    cfg.BoolOpt('publish_errors',
+                default=False,
+                help='publish error events'),
+
+    # NOTE(mikal): there are two options here because sometimes we are handed
+    # a full instance (and could include more information), and other times we
+    # are just handed a UUID for the instance.
+    cfg.StrOpt('instance_format',
+               default='[instance: %(uuid)s] ',
+               help='If an instance is passed with the log message, format '
+                    'it like this'),
+    cfg.StrOpt('instance_uuid_format',
+               default='[instance: %(uuid)s] ',
+               help='If an instance UUID is passed with the log message, '
+                    'format it like this'),
+    ]
+
+
+generic_log_opts = [
+    cfg.StrOpt('logdir',
+               default=None,
+               help='Log output to a per-service log file in named directory'),
+    cfg.StrOpt('logfile',
+               default=None,
+               help='Log output to a named file'),
+    cfg.BoolOpt('use_stderr',
+                default=True,
+                help='Log output to standard error'),
+    cfg.StrOpt('logfile_mode',
+               default='0644',
+               help='Default file mode used when creating log files'),
+    ]
+
+
+CONF = cfg.CONF
+CONF.register_opts(generic_log_opts)
+CONF.register_opts(log_opts)
+
+# our new audit level
+# NOTE(jkoelker) Since we synthesized an audit level, make the logging
+#                module aware of it so it acts like other levels.
+logging.AUDIT = logging.INFO + 1
+logging.addLevelName(logging.AUDIT, 'AUDIT')
+
+
+try:
+    NullHandler = logging.NullHandler
+except AttributeError:  # NOTE(jkoelker) NullHandler added in Python 2.7
+    class NullHandler(logging.Handler):
+        def handle(self, record):
+            pass
+
+        def emit(self, record):
+            pass
+
+        def createLock(self):
+            self.lock = None
+
+
+def _dictify_context(context):
+    if context is None:
+        return None
+    if not isinstance(context, dict) and getattr(context, 'to_dict', None):
+        context = context.to_dict()
+    return context
+
+
+def _get_binary_name():
+    return os.path.basename(inspect.stack()[-1][1])
+
+
+def _get_log_file_path(binary=None):
+    logfile = CONF.log_file or CONF.logfile
+    logdir = CONF.log_dir or CONF.logdir
+
+    if logfile and not logdir:
+        return logfile
+
+    if logfile and logdir:
+        return os.path.join(logdir, logfile)
+
+    if logdir:
+        binary = binary or _get_binary_name()
+        return '%s.log' % (os.path.join(logdir, binary),)
+
+
+class ContextAdapter(logging.LoggerAdapter):
+    warn = logging.LoggerAdapter.warning
+
+    def __init__(self, logger, project_name, version_string):
+        self.logger = logger
+        self.project = project_name
+        self.version = version_string
+
+    def audit(self, msg, *args, **kwargs):
+        self.log(logging.AUDIT, msg, *args, **kwargs)
+
+    def process(self, msg, kwargs):
+        if 'extra' not in kwargs:
+            kwargs['extra'] = {}
+        extra = kwargs['extra']
+
+        context = kwargs.pop('context', None)
+        if not context:
+            context = getattr(local.store, 'context', None)
+        if context:
+            extra.update(_dictify_context(context))
+
+        instance = kwargs.pop('instance', None)
+        instance_extra = ''
+        if instance:
+            instance_extra = CONF.instance_format % instance
+        else:
+            instance_uuid = kwargs.pop('instance_uuid', None)
+            if instance_uuid:
+                instance_extra = (CONF.instance_uuid_format
+                                  % {'uuid': instance_uuid})
+        extra.update({'instance': instance_extra})
+
+        extra.update({"project": self.project})
+        extra.update({"version": self.version})
+        extra['extra'] = extra.copy()
+        return msg, kwargs
+
+
+class JSONFormatter(logging.Formatter):
+    def __init__(self, fmt=None, datefmt=None):
+        # NOTE(jkoelker) we ignore the fmt argument, but its still there
+        #                since logging.config.fileConfig passes it.
+        self.datefmt = datefmt
+
+    def formatException(self, ei, strip_newlines=True):
+        lines = traceback.format_exception(*ei)
+        if strip_newlines:
+            lines = [itertools.ifilter(lambda x: x,
+                                      line.rstrip().splitlines())
+                    for line in lines]
+            lines = list(itertools.chain(*lines))
+        return lines
+
+    def format(self, record):
+        message = {'message': record.getMessage(),
+                   'asctime': self.formatTime(record, self.datefmt),
+                   'name': record.name,
+                   'msg': record.msg,
+                   'args': record.args,
+                   'levelname': record.levelname,
+                   'levelno': record.levelno,
+                   'pathname': record.pathname,
+                   'filename': record.filename,
+                   'module': record.module,
+                   'lineno': record.lineno,
+                   'funcname': record.funcName,
+                   'created': record.created,
+                   'msecs': record.msecs,
+                   'relative_created': record.relativeCreated,
+                   'thread': record.thread,
+                   'thread_name': record.threadName,
+                   'process_name': record.processName,
+                   'process': record.process,
+                   'traceback': None}
+
+        if hasattr(record, 'extra'):
+            message['extra'] = record.extra
+
+        if record.exc_info:
+            message['traceback'] = self.formatException(record.exc_info)
+
+        return jsonutils.dumps(message)
+
+
+class PublishErrorsHandler(logging.Handler):
+    def emit(self, record):
+        if 'list_notifier_drivers' in CONF:
+            if ('heat.openstack.common.notifier.log_notifier' in
+                CONF.list_notifier_drivers):
+                return
+        notifier.api.notify(None, 'error.publisher',
+                                 'error_notification',
+                                 notifier.api.ERROR,
+                                 dict(error=record.msg))
+
+
+def handle_exception(type, value, tb):
+    extra = {}
+    if CONF.verbose:
+        extra['exc_info'] = (type, value, tb)
+    getLogger().critical(str(value), **extra)
+
+
+def setup(product_name):
+    """Setup logging."""
+    sys.excepthook = handle_exception
+
+    if CONF.log_config:
+        try:
+            logging.config.fileConfig(CONF.log_config)
+        except Exception:
+            traceback.print_exc()
+            raise
+    else:
+        _setup_logging_from_conf(product_name)
+
+
+def _find_facility_from_conf():
+    facility_names = logging.handlers.SysLogHandler.facility_names
+    facility = getattr(logging.handlers.SysLogHandler,
+                       CONF.syslog_log_facility,
+                       None)
+
+    if facility is None and CONF.syslog_log_facility in facility_names:
+        facility = facility_names.get(CONF.syslog_log_facility)
+
+    if facility is None:
+        valid_facilities = facility_names.keys()
+        consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON',
+                  'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS',
+                  'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP',
+                  'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3',
+                  'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7']
+        valid_facilities.extend(consts)
+        raise TypeError(_('syslog facility must be one of: %s') %
+                        ', '.join("'%s'" % fac
+                                  for fac in valid_facilities))
+
+    return facility
+
+
+def _setup_logging_from_conf(product_name):
+    log_root = getLogger(product_name).logger
+    for handler in log_root.handlers:
+        log_root.removeHandler(handler)
+
+    if CONF.use_syslog:
+        facility = _find_facility_from_conf()
+        syslog = logging.handlers.SysLogHandler(address='/dev/log',
+                                                facility=facility)
+        log_root.addHandler(syslog)
+
+    logpath = _get_log_file_path()
+    if logpath:
+        filelog = logging.handlers.WatchedFileHandler(logpath)
+        log_root.addHandler(filelog)
+
+        mode = int(CONF.logfile_mode, 8)
+        st = os.stat(logpath)
+        if st.st_mode != (stat.S_IFREG | mode):
+            os.chmod(logpath, mode)
+
+    if CONF.use_stderr:
+        streamlog = ColorHandler()
+        log_root.addHandler(streamlog)
+
+    elif not CONF.log_file:
+        # pass sys.stdout as a positional argument
+        # python2.6 calls the argument strm, in 2.7 it's stream
+        streamlog = logging.StreamHandler(sys.stdout)
+        log_root.addHandler(streamlog)
+
+    if CONF.publish_errors:
+        log_root.addHandler(PublishErrorsHandler(logging.ERROR))
+
+    for handler in log_root.handlers:
+        datefmt = CONF.log_date_format
+        if CONF.log_format:
+            handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
+                                                   datefmt=datefmt))
+        handler.setFormatter(LegacyFormatter(datefmt=datefmt))
+
+    if CONF.verbose or CONF.debug:
+        log_root.setLevel(logging.DEBUG)
+    else:
+        log_root.setLevel(logging.INFO)
+
+    level = logging.NOTSET
+    for pair in CONF.default_log_levels:
+        mod, _sep, level_name = pair.partition('=')
+        level = logging.getLevelName(level_name)
+        logger = logging.getLogger(mod)
+        logger.setLevel(level)
+        for handler in log_root.handlers:
+            logger.addHandler(handler)
+
+    # NOTE(jkoelker) Clear the handlers for the root logger that was setup
+    #                by basicConfig in nova/__init__.py and install the
+    #                NullHandler.
+    root = logging.getLogger()
+    for handler in root.handlers:
+        root.removeHandler(handler)
+    handler = NullHandler()
+    handler.setFormatter(logging.Formatter())
+    root.addHandler(handler)
+
+
+_loggers = {}
+
+
+def getLogger(name='unknown', version='unknown'):
+    if name not in _loggers:
+        _loggers[name] = ContextAdapter(logging.getLogger(name),
+                                        name,
+                                        version)
+    return _loggers[name]
+
+
+class WritableLogger(object):
+    """A thin wrapper that responds to `write` and logs."""
+
+    def __init__(self, logger, level=logging.INFO):
+        self.logger = logger
+        self.level = level
+
+    def write(self, msg):
+        self.logger.log(self.level, msg)
+
+
+class LegacyFormatter(logging.Formatter):
+    """A context.RequestContext aware formatter configured through flags.
+
+    The flags used to set format strings are: logging_context_format_string
+    and logging_default_format_string.  You can also specify
+    logging_debug_format_suffix to append extra formatting if the log level is
+    debug.
+
+    For information about what variables are available for the formatter see:
+    http://docs.python.org/library/logging.html#formatter
+
+    """
+
+    def format(self, record):
+        """Uses contextstring if request_id is set, otherwise default."""
+        if 'instance' not in record.__dict__:
+            record.__dict__['instance'] = ''
+
+        if record.__dict__.get('request_id', None):
+            self._fmt = CONF.logging_context_format_string
+        else:
+            self._fmt = CONF.logging_default_format_string
+
+        if (record.levelno == logging.DEBUG and
+            CONF.logging_debug_format_suffix):
+            self._fmt += " " + CONF.logging_debug_format_suffix
+
+        # Cache this on the record, Logger will respect our formated copy
+        if record.exc_info:
+            record.exc_text = self.formatException(record.exc_info, record)
+        return logging.Formatter.format(self, record)
+
+    def formatException(self, exc_info, record=None):
+        """Format exception output with CONF.logging_exception_prefix."""
+        if not record:
+            return logging.Formatter.formatException(self, exc_info)
+
+        stringbuffer = cStringIO.StringIO()
+        traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
+                                  None, stringbuffer)
+        lines = stringbuffer.getvalue().split('\n')
+        stringbuffer.close()
+
+        if CONF.logging_exception_prefix.find('%(asctime)') != -1:
+            record.asctime = self.formatTime(record, self.datefmt)
+
+        formatted_lines = []
+        for line in lines:
+            pl = CONF.logging_exception_prefix % record.__dict__
+            fl = '%s%s' % (pl, line)
+            formatted_lines.append(fl)
+        return '\n'.join(formatted_lines)
+
+
+class ColorHandler(logging.StreamHandler):
+    LEVEL_COLORS = {
+        logging.DEBUG: '\033[00;32m',  # GREEN
+        logging.INFO: '\033[00;36m',  # CYAN
+        logging.AUDIT: '\033[01;36m',  # BOLD CYAN
+        logging.WARN: '\033[01;33m',  # BOLD YELLOW
+        logging.ERROR: '\033[01;31m',  # BOLD RED
+        logging.CRITICAL: '\033[01;31m',  # BOLD RED
+    }
+
+    def format(self, record):
+        record.color = self.LEVEL_COLORS[record.levelno]
+        return logging.StreamHandler.format(self, record)

heat/openstack/common/notifier/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

heat/openstack/common/notifier/api.py

@@ -0,0 +1,142 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import inspect
+import uuid
+
+from heat.openstack.common import cfg
+from heat.openstack.common import context
+from heat.openstack.common.gettextutils import _
+from heat.openstack.common import importutils
+from heat.openstack.common import jsonutils
+from heat.openstack.common import log as logging
+from heat.openstack.common import timeutils
+
+
+LOG = logging.getLogger(__name__)
+
+notifier_opts = [
+    cfg.StrOpt('notification_driver',
+               default='heat.openstack.common.notifier.no_op_notifier',
+               help='Default driver for sending notifications'),
+    cfg.StrOpt('default_notification_level',
+               default='INFO',
+               help='Default notification level for outgoing notifications'),
+    cfg.StrOpt('default_publisher_id',
+               default='$host',
+               help='Default publisher_id for outgoing notifications'),
+    ]
+
+CONF = cfg.CONF
+CONF.register_opts(notifier_opts)
+
+WARN = 'WARN'
+INFO = 'INFO'
+ERROR = 'ERROR'
+CRITICAL = 'CRITICAL'
+DEBUG = 'DEBUG'
+
+log_levels = (DEBUG, WARN, INFO, ERROR, CRITICAL)
+
+
+class BadPriorityException(Exception):
+    pass
+
+
+def notify_decorator(name, fn):
+    """ decorator for notify which is used from utils.monkey_patch()
+
+        :param name: name of the function
+        :param function: - object of the function
+        :returns: function -- decorated function
+
+    """
+    def wrapped_func(*args, **kwarg):
+        body = {}
+        body['args'] = []
+        body['kwarg'] = {}
+        for arg in args:
+            body['args'].append(arg)
+        for key in kwarg:
+            body['kwarg'][key] = kwarg[key]
+
+        ctxt = context.get_context_from_function_and_args(fn, args, kwarg)
+        notify(ctxt,
+               CONF.default_publisher_id,
+               name,
+               CONF.default_notification_level,
+               body)
+        return fn(*args, **kwarg)
+    return wrapped_func
+
+
+def publisher_id(service, host=None):
+    if not host:
+        host = CONF.host
+    return "%s.%s" % (service, host)
+
+
+def notify(context, publisher_id, event_type, priority, payload):
+    """Sends a notification using the specified driver
+
+    :param publisher_id: the source worker_type.host of the message
+    :param event_type:   the literal type of event (ex. Instance Creation)
+    :param priority:     patterned after the enumeration of Python logging
+                         levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL)
+    :param payload:       A python dictionary of attributes
+
+    Outgoing message format includes the above parameters, and appends the
+    following:
+
+    message_id
+      a UUID representing the id for this notification
+
+    timestamp
+      the GMT timestamp the notification was sent at
+
+    The composite message will be constructed as a dictionary of the above
+    attributes, which will then be sent via the transport mechanism defined
+    by the driver.
+
+    Message example::
+
+        {'message_id': str(uuid.uuid4()),
+         'publisher_id': 'compute.host1',
+         'timestamp': timeutils.utcnow(),
+         'priority': 'WARN',
+         'event_type': 'compute.create_instance',
+         'payload': {'instance_id': 12, ... }}
+
+    """
+    if priority not in log_levels:
+        raise BadPriorityException(
+                 _('%s not in valid priorities') % priority)
+
+    # Ensure everything is JSON serializable.
+    payload = jsonutils.to_primitive(payload, convert_instances=True)
+
+    driver = importutils.import_module(CONF.notification_driver)
+    msg = dict(message_id=str(uuid.uuid4()),
+                   publisher_id=publisher_id,
+                   event_type=event_type,
+                   priority=priority,
+                   payload=payload,
+                   timestamp=str(timeutils.utcnow()))
+    try:
+        driver.notify(context, msg)
+    except Exception, e:
+        LOG.exception(_("Problem '%(e)s' attempting to "
+                        "send to notification system. Payload=%(payload)s") %
+                        locals())

heat/openstack/common/notifier/list_notifier.py

@@ -0,0 +1,117 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from heat.openstack.common import cfg
+from heat.openstack.common.gettextutils import _
+from heat.openstack.common import importutils
+from heat.openstack.common import log as logging
+
+
+list_notifier_drivers_opt = cfg.MultiStrOpt('list_notifier_drivers',
+        default=['heat.openstack.common.notifier.no_op_notifier'],
+        help='List of drivers to send notifications')
+
+CONF = cfg.CONF
+CONF.register_opt(list_notifier_drivers_opt)
+
+LOG = logging.getLogger(__name__)
+
+drivers = None
+
+
+class ImportFailureNotifier(object):
+    """Noisily re-raises some exception over-and-over when notify is called."""
+
+    def __init__(self, exception):
+        self.exception = exception
+
+    def notify(self, context, message):
+        raise self.exception
+
+
+def _get_drivers():
+    """Instantiates and returns drivers based on the flag values."""
+    global drivers
+    if drivers is None:
+        drivers = []
+        for notification_driver in CONF.list_notifier_drivers:
+            try:
+                drivers.append(importutils.import_module(notification_driver))
+            except ImportError as e:
+                drivers.append(ImportFailureNotifier(e))
+    return drivers
+
+
+def add_driver(notification_driver):
+    """Add a notification driver at runtime."""
+    # Make sure the driver list is initialized.
+    _get_drivers()
+    if isinstance(notification_driver, basestring):
+        # Load and add
+        try:
+            drivers.append(importutils.import_module(notification_driver))
+        except ImportError as e:
+            drivers.append(ImportFailureNotifier(e))
+    else:
+        # Driver is already loaded; just add the object.
+        drivers.append(notification_driver)
+
+
+def _object_name(obj):
+    name = []
+    if hasattr(obj, '__module__'):
+        name.append(obj.__module__)
+    if hasattr(obj, '__name__'):
+        name.append(obj.__name__)
+    else:
+        name.append(obj.__class__.__name__)
+    return '.'.join(name)
+
+
+def remove_driver(notification_driver):
+    """Remove a notification driver at runtime."""
+    # Make sure the driver list is initialized.
+    _get_drivers()
+    removed = False
+    if notification_driver in drivers:
+        # We're removing an object.  Easy.
+        drivers.remove(notification_driver)
+        removed = True
+    else:
+        # We're removing a driver by name.  Search for it.
+        for driver in drivers:
+            if _object_name(driver) == notification_driver:
+                drivers.remove(driver)
+                removed = True
+
+    if not removed:
+        raise ValueError("Cannot remove; %s is not in list" %
+                         notification_driver)
+
+
+def notify(context, message):
+    """Passes notification to multiple notifiers in a list."""
+    for driver in _get_drivers():
+        try:
+            driver.notify(context, message)
+        except Exception as e:
+            LOG.exception(_("Problem '%(e)s' attempting to send to "
+                            "notification driver %(driver)s."), locals())
+
+
+def _reset_drivers():
+    """Used by unit tests to reset the drivers."""
+    global drivers
+    drivers = None

heat/openstack/common/notifier/log_notifier.py

@@ -0,0 +1,35 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from heat.openstack.common import cfg
+from heat.openstack.common import jsonutils
+from heat.openstack.common import log as logging
+
+
+CONF = cfg.CONF
+
+
+def notify(_context, message):
+    """Notifies the recipient of the desired event given the model.
+    Log notifications using openstack's default logging system"""
+
+    priority = message.get('priority',
+                           CONF.default_notification_level)
+    priority = priority.lower()
+    logger = logging.getLogger(
+            'heat.openstack.common.notification.%s' %
+            message['event_type'])
+    getattr(logger, priority)(jsonutils.dumps(message))

heat/openstack/common/notifier/no_op_notifier.py

@@ -0,0 +1,19 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+def notify(_context, message):
+    """Notifies the recipient of the desired event given the model"""
+    pass

heat/openstack/common/notifier/rabbit_notifier.py

@@ -0,0 +1,46 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from heat.openstack.common import cfg
+from heat.openstack.common import context as req_context
+from heat.openstack.common.gettextutils import _
+from heat.openstack.common import log as logging
+from heat.openstack.common import rpc
+
+LOG = logging.getLogger(__name__)
+
+notification_topic_opt = cfg.ListOpt('notification_topics',
+        default=['notifications', ],
+        help='AMQP topic used for openstack notifications')
+
+CONF = cfg.CONF
+CONF.register_opt(notification_topic_opt)
+
+
+def notify(context, message):
+    """Sends a notification to the RabbitMQ"""
+    if not context:
+        context = req_context.get_admin_context()
+    priority = message.get('priority',
+                           CONF.default_notification_level)
+    priority = priority.lower()
+    for topic in CONF.notification_topics:
+        topic = '%s.%s' % (topic, priority)
+        try:
+            rpc.notify(context, topic, message)
+        except Exception, e:
+            LOG.exception(_("Could not send notification to %(topic)s. "
+                            "Payload=%(message)s"), locals())

heat/openstack/common/notifier/test_notifier.py

@@ -0,0 +1,22 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+NOTIFICATIONS = []
+
+
+def notify(_context, message):
+    """Test notifier, stores notifications in memory for unittests."""
+    NOTIFICATIONS.append(message)

heat/rpc/__init__.py

@@ -17,10 +17,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import logging
 from heat.openstack.common import cfg
 from heat.openstack.common import importutils
 
+from heat.openstack.common import log as logging
+
 LOG = logging.getLogger(__name__)
 
 

heat/rpc/amqp.py

@@ -26,11 +26,12 @@ AMQP, but is deprecated and predates this code.
 """
 
 import inspect
-import logging
 import sys
 import traceback
 import uuid
 
+from heat.openstack.common import log as logging
+
 from eventlet import greenpool
 from eventlet import pools
 

heat/rpc/common.py

@@ -18,13 +18,12 @@
 #    under the License.
 
 import copy
-import logging
 
 from heat.openstack.common import cfg
 from heat.openstack.common import exception
+from heat.openstack.common import log as logging
 from heat.common import config
 
-
 LOG = logging.getLogger(__name__)
 
 

heat/rpc/impl_qpid.py

@@ -19,7 +19,6 @@ import itertools
 import time
 import uuid
 import json
-import logging
 
 import eventlet
 import greenlet
@@ -31,6 +30,8 @@ from heat.openstack.common import cfg
 from heat.rpc import amqp as rpc_amqp
 from heat.rpc import common as rpc_common
 
+from heat.openstack.common import log as logging
+
 LOG = logging.getLogger(__name__)
 
 

heat/service.py

@@ -23,11 +23,11 @@ import inspect
 import os
 
 import eventlet
-import logging
 import greenlet
 
 from heat.openstack.common import cfg
 from heat.openstack.common import importutils
+from heat.openstack.common import log as logging
 
 from heat.common import utils as heat_utils
 from heat.common import exception

heat/utils.py

@@ -20,11 +20,13 @@ import sys
 import base64
 from lxml import etree
 import re
-import logging
 
 from glance import client as glance_client
 from heat.common import exception
 
+from heat.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
 
 SUCCESS = 0
 FAILURE = 1
@@ -39,14 +41,14 @@ def catch_error(action):
                 ret = func(*arguments, **kwargs)
                 return SUCCESS if ret is None else ret
             except exception.NotAuthorized:
-                logging.error("Not authorized to make this request. Check " +
+                LOG.error("Not authorized to make this request. Check " +
                       "your credentials (OS_USERNAME, OS_PASSWORD, " +
                       "OS_TENANT_NAME, OS_AUTH_URL and OS_AUTH_STRATEGY).")
                 return FAILURE
             except exception.ClientConfigurationError:
                 raise
             except exception.KeystoneError, e:
-                logging.error("Keystone did not finish the authentication and "
+                LOG.error("Keystone did not finish the authentication and "
                               "returned the following message:\n\n%s"
                               % e.message)
                 return FAILURE
@@ -54,10 +56,10 @@ def catch_error(action):
                 options = arguments[0]
                 if options.debug:
                     raise
-                logging.error("Failed to %s. Got error:" % action)
+                LOG.error("Failed to %s. Got error:" % action)
                 pieces = unicode(e).split('\n')
                 for piece in pieces:
-                    logging.error(piece)
+                    LOG.error(piece)
                 return FAILURE
 
         return wrapper

openstack-common.conf

@@ -1,7 +1,7 @@
 [DEFAULT]
 
 # The list of modules to copy from openstack-common
-modules=gettextutils,cfg,local,iniparser,utils,exception,timeutils,importutils,setup
+modules=gettextutils,cfg,local,iniparser,utils,exception,timeutils,importutils,setup,log,jsonutils,notifier
 
 # The base module to hold the copy of openstack.common
 base=heat