diff --git a/bin/heat-api b/bin/heat-api index 2fe28816be..d588cd0a69 100755 --- a/bin/heat-api +++ b/bin/heat-api @@ -33,12 +33,13 @@ if os.path.exists(os.path.join(possible_topdir, 'heat', '__init__.py')): gettext.install('heat', unicode=1) -import logging from heat import rpc from heat.common import config from heat.common import wsgi from paste import httpserver +from heat.openstack.common import log as logging + LOG = logging.getLogger('heat.api') if __name__ == '__main__': diff --git a/bin/heat-engine b/bin/heat-engine index dba0a6c1bd..9df2929c2a 100755 --- a/bin/heat-engine +++ b/bin/heat-engine @@ -26,7 +26,8 @@ eventlet.monkey_patch() import os import sys -import logging + +from heat.openstack.common import log as logging # If ../heat/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... diff --git a/bin/heat-metadata b/bin/heat-metadata index fb7802beba..cd3a57dcb6 100755 --- a/bin/heat-metadata +++ b/bin/heat-metadata @@ -33,13 +33,14 @@ if os.path.exists(os.path.join(possible_topdir, 'heat', '__init__.py')): gettext.install('heat', unicode=1) -import logging from heat import rpc from heat.common import config from heat.common import wsgi from heat.common import context from paste import httpserver +from heat.openstack.common import log as logging + LOG = logging.getLogger('heat.metadata') diff --git a/heat/api/middleware/version_negotiation.py b/heat/api/middleware/version_negotiation.py index a4efcc6057..8f2826df6f 100644 --- a/heat/api/middleware/version_negotiation.py +++ b/heat/api/middleware/version_negotiation.py @@ -19,9 +19,10 @@ and/or Accept headers and attempts to negotiate an API controller to return """ -import logging import re +from heat.openstack.common import log as logging + from heat.api import versions from heat.common import wsgi diff --git a/heat/api/v1/__init__.py b/heat/api/v1/__init__.py index e5f1d1175d..9b7fa2dc73 100644 --- a/heat/api/v1/__init__.py +++ b/heat/api/v1/__init__.py @@ -16,7 +16,6 @@ import json import urlparse import httplib -import logging import routes import gettext @@ -31,6 +30,8 @@ from heat import utils from heat.common import context from heat.api.v1 import exception +from heat.openstack.common import log as logging + logger = logging.getLogger(__name__) diff --git a/heat/api/v1/stacks.py b/heat/api/v1/stacks.py index dfa0e046b2..1f90105337 100644 --- a/heat/api/v1/stacks.py +++ b/heat/api/v1/stacks.py @@ -18,7 +18,6 @@ """ import httplib import json -import logging import os import socket import sys @@ -33,6 +32,7 @@ from heat import utils from heat import rpc import heat.rpc.common as rpc_common +from heat.openstack.common import log as logging logger = logging.getLogger('heat.api.v1.stacks') diff --git a/heat/client.py b/heat/client.py index 4a5dc1f03c..078dc69280 100644 --- a/heat/client.py +++ b/heat/client.py @@ -18,11 +18,13 @@ Client classes for callers of a heat system """ from lxml import etree -import logging import os from heat.common import client as base_client from heat.common import exception from heat.cloudformations import * + +from heat.openstack.common import log as logging + logger = logging.getLogger(__name__) diff --git a/heat/db/sqlalchemy/session.py b/heat/db/sqlalchemy/session.py index dcdd6efbb1..f4f783d155 100644 --- a/heat/db/sqlalchemy/session.py +++ b/heat/db/sqlalchemy/session.py @@ -14,11 +14,12 @@ """Session Handling for SQLAlchemy backend.""" -import logging import sqlalchemy.interfaces import sqlalchemy.orm from sqlalchemy.exc import DisconnectionError +from heat.openstack.common import log as logging + from heat.db import api as db_api from heat.openstack.common import cfg diff --git a/heat/engine/api.py b/heat/engine/api.py index fc81eb8695..a87a3fb805 100644 --- a/heat/engine/api.py +++ b/heat/engine/api.py @@ -13,10 +13,10 @@ # under the License. import re -import logging from heat.common import utils as heat_utils from heat.engine import parser +from heat.openstack.common import log as logging logger = logging.getLogger('heat.engine.manager') diff --git a/heat/engine/auth.py b/heat/engine/auth.py index c8b27cfdc8..bd547b0c56 100644 --- a/heat/engine/auth.py +++ b/heat/engine/auth.py @@ -13,7 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import json import httplib import urlparse @@ -22,6 +21,7 @@ from novaclient.exceptions import BadRequest from novaclient.exceptions import NotFound from novaclient.exceptions import AuthorizationFailure from heat.common import context +from heat.openstack.common import log as logging logger = logging.getLogger('heat.engine.auth') diff --git a/heat/engine/autoscaling.py b/heat/engine/autoscaling.py index 212487904d..ccecf21e0e 100644 --- a/heat/engine/autoscaling.py +++ b/heat/engine/autoscaling.py @@ -14,7 +14,6 @@ # under the License. import eventlet -import logging import json import os @@ -23,6 +22,8 @@ from heat.db import api as db_api from heat.engine import instance from heat.engine.resources import Resource +from heat.openstack.common import log as logging + logger = logging.getLogger('heat.engine.autoscaling') diff --git a/heat/engine/checkeddict.py b/heat/engine/checkeddict.py index 7893a66894..554aab08d3 100644 --- a/heat/engine/checkeddict.py +++ b/heat/engine/checkeddict.py @@ -15,7 +15,8 @@ import collections import re -import logging + +from heat.openstack.common import log as logging logger = logging.getLogger('heat.engine.checkeddict') diff --git a/heat/engine/cloud_watch.py b/heat/engine/cloud_watch.py index 451986ea95..68966286c6 100644 --- a/heat/engine/cloud_watch.py +++ b/heat/engine/cloud_watch.py @@ -14,7 +14,6 @@ # under the License. import eventlet -import logging import json import os @@ -22,6 +21,8 @@ from heat.common import exception from heat.db import api as db_api from heat.engine.resources import Resource +from heat.openstack.common import log as logging + logger = logging.getLogger('heat.engine.cloud_watch') diff --git a/heat/engine/eip.py b/heat/engine/eip.py index 5aa032faba..de22eeae5c 100644 --- a/heat/engine/eip.py +++ b/heat/engine/eip.py @@ -13,12 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - from heat.common import exception from heat.engine.resources import Resource from novaclient.exceptions import NotFound +from heat.openstack.common import log as logging + logger = logging.getLogger('heat.engine.eip') diff --git a/heat/engine/instance.py b/heat/engine/instance.py index a1c794de5f..351886f837 100644 --- a/heat/engine/instance.py +++ b/heat/engine/instance.py @@ -14,7 +14,6 @@ # under the License. import eventlet -import logging import os import json import sys @@ -26,6 +25,8 @@ import heat from heat.engine import resources from heat.common import exception +from heat.openstack.common import log as logging + logger = logging.getLogger('heat.engine.instance') diff --git a/heat/engine/loadbalancer.py b/heat/engine/loadbalancer.py index fe2705337c..dfe473695b 100644 --- a/heat/engine/loadbalancer.py +++ b/heat/engine/loadbalancer.py @@ -15,7 +15,6 @@ import urllib2 import json -import logging from heat.common import exception from heat.engine import stack @@ -23,6 +22,8 @@ from heat.db import api as db_api from heat.engine import parser from novaclient.exceptions import NotFound +from heat.openstack.common import log as logging + logger = logging.getLogger(__file__) lb_template = ''' diff --git a/heat/engine/manager.py b/heat/engine/manager.py index 9ea607547d..d0eb2b5cd6 100644 --- a/heat/engine/manager.py +++ b/heat/engine/manager.py @@ -16,7 +16,6 @@ from copy import deepcopy import datetime -import logging import webob import json import urlparse @@ -33,7 +32,9 @@ from heat.engine import parser from heat.engine import resources from heat.engine import watchrule from heat.engine import auth + from heat.openstack.common import timeutils +from heat.openstack.common import log as logging from novaclient.v1_1 import client from novaclient.exceptions import BadRequest diff --git a/heat/engine/parser.py b/heat/engine/parser.py index c91792773e..230b522e2b 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -17,7 +17,6 @@ import eventlet import json import functools import copy -import logging from heat.common import exception from heat.engine import checkeddict @@ -25,6 +24,7 @@ from heat.engine import dependencies from heat.engine import resources from heat.db import api as db_api +from heat.openstack.common import log as logging logger = logging.getLogger('heat.engine.parser') diff --git a/heat/engine/resources.py b/heat/engine/resources.py index a7a70d8529..db2608e1bf 100644 --- a/heat/engine/resources.py +++ b/heat/engine/resources.py @@ -15,7 +15,6 @@ import base64 from datetime import datetime -import logging from novaclient.v1_1 import client as nc from keystoneclient.v2_0 import client as kc @@ -26,6 +25,8 @@ from heat.db import api as db_api from heat.engine import checkeddict from heat.engine import auth +from heat.openstack.common import log as logging + logger = logging.getLogger('heat.engine.resources') diff --git a/heat/engine/security_group.py b/heat/engine/security_group.py index edc913ed3e..65a01dda5e 100644 --- a/heat/engine/security_group.py +++ b/heat/engine/security_group.py @@ -13,12 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. -import logging from novaclient.exceptions import BadRequest from novaclient.exceptions import NotFound from heat.common import exception from heat.engine.resources import Resource +from heat.openstack.common import log as logging + logger = logging.getLogger('heat.engine.security_group') diff --git a/heat/engine/stack.py b/heat/engine/stack.py index cbc46aed4c..145c8d3bdb 100644 --- a/heat/engine/stack.py +++ b/heat/engine/stack.py @@ -15,13 +15,14 @@ import urllib2 import json -import logging from heat.common import exception from heat.engine.resources import Resource from heat.db import api as db_api from heat.engine import parser +from heat.openstack.common import log as logging + logger = logging.getLogger(__file__) diff --git a/heat/engine/user.py b/heat/engine/user.py index 083fb077a9..be09b48534 100644 --- a/heat/engine/user.py +++ b/heat/engine/user.py @@ -14,10 +14,10 @@ # under the License. import eventlet -import logging from heat.common import exception from heat.engine.resources import Resource +from heat.openstack.common import log as logging logger = logging.getLogger('heat.engine.user') diff --git a/heat/engine/volume.py b/heat/engine/volume.py index d06c77726c..dee0debe98 100644 --- a/heat/engine/volume.py +++ b/heat/engine/volume.py @@ -14,7 +14,7 @@ # under the License. import eventlet -import logging +from heat.openstack.common import log as logging import re from heat.common import exception diff --git a/heat/engine/wait_condition.py b/heat/engine/wait_condition.py index a391925afb..fc908a573b 100644 --- a/heat/engine/wait_condition.py +++ b/heat/engine/wait_condition.py @@ -14,12 +14,13 @@ # under the License. import eventlet -import logging import json from heat.common import exception from heat.engine import resources +from heat.openstack.common import log as logging + logger = logging.getLogger('heat.engine.wait_condition') diff --git a/heat/engine/watchrule.py b/heat/engine/watchrule.py index e11ccfd19c..4767f5bd0e 100644 --- a/heat/engine/watchrule.py +++ b/heat/engine/watchrule.py @@ -15,7 +15,7 @@ import datetime -import logging +from heat.openstack.common import log as logging from heat.openstack.common import timeutils logger = logging.getLogger('heat.engine.watchrule') diff --git a/heat/manager.py b/heat/manager.py index 8940fdc630..3739f18ef4 100644 --- a/heat/manager.py +++ b/heat/manager.py @@ -53,11 +53,12 @@ This module provides Manager, a base class for managers. """ -import logging - from heat import version from heat.common import config +from heat.openstack.common import log as logging + + FLAGS = config.FLAGS LOG = logging.getLogger(__name__) diff --git a/heat/metadata/api/v1/__init__.py b/heat/metadata/api/v1/__init__.py index 2a629ef415..7489929171 100644 --- a/heat/metadata/api/v1/__init__.py +++ b/heat/metadata/api/v1/__init__.py @@ -13,12 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import routes from heat.common import wsgi from heat.metadata.api.v1 import metadata +from heat.openstack.common import log as logging + class API(wsgi.Router): """ diff --git a/heat/openstack/common/jsonutils.py b/heat/openstack/common/jsonutils.py new file mode 100644 index 0000000000..11b7e1efb5 --- /dev/null +++ b/heat/openstack/common/jsonutils.py @@ -0,0 +1,144 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +''' +JSON related utilities. + +This module provides a few things: + + 1) A handy function for getting an object down to something that can be + JSON serialized. See to_primitive(). + + 2) Wrappers around loads() and dumps(). The dumps() wrapper will + automatically use to_primitive() for you if needed. + + 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson + is available. +''' + + +import datetime +import inspect +import itertools +import json +import xmlrpclib + + +def to_primitive(value, convert_instances=False, level=0): + """Convert a complex object into primitives. + + Handy for JSON serialization. We can optionally handle instances, + but since this is a recursive function, we could have cyclical + data structures. + + To handle cyclical data structures we could track the actual objects + visited in a set, but not all objects are hashable. Instead we just + track the depth of the object inspections and don't go too deep. + + Therefore, convert_instances=True is lossy ... be aware. + + """ + nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod, + inspect.isfunction, inspect.isgeneratorfunction, + inspect.isgenerator, inspect.istraceback, inspect.isframe, + inspect.iscode, inspect.isbuiltin, inspect.isroutine, + inspect.isabstract] + for test in nasty: + if test(value): + return unicode(value) + + # value of itertools.count doesn't get caught by inspects + # above and results in infinite loop when list(value) is called. + if type(value) == itertools.count: + return unicode(value) + + # FIXME(vish): Workaround for LP bug 852095. Without this workaround, + # tests that raise an exception in a mocked method that + # has a @wrap_exception with a notifier will fail. If + # we up the dependency to 0.5.4 (when it is released) we + # can remove this workaround. + if getattr(value, '__module__', None) == 'mox': + return 'mock' + + if level > 3: + return '?' + + # The try block may not be necessary after the class check above, + # but just in case ... + try: + # It's not clear why xmlrpclib created their own DateTime type, but + # for our purposes, make it a datetime type which is explicitly + # handled + if isinstance(value, xmlrpclib.DateTime): + value = datetime.datetime(*tuple(value.timetuple())[:6]) + + if isinstance(value, (list, tuple)): + o = [] + for v in value: + o.append(to_primitive(v, convert_instances=convert_instances, + level=level)) + return o + elif isinstance(value, dict): + o = {} + for k, v in value.iteritems(): + o[k] = to_primitive(v, convert_instances=convert_instances, + level=level) + return o + elif isinstance(value, datetime.datetime): + return str(value) + elif hasattr(value, 'iteritems'): + return to_primitive(dict(value.iteritems()), + convert_instances=convert_instances, + level=level) + elif hasattr(value, '__iter__'): + return to_primitive(list(value), level) + elif convert_instances and hasattr(value, '__dict__'): + # Likely an instance of something. Watch for cycles. + # Ignore class member vars. + return to_primitive(value.__dict__, + convert_instances=convert_instances, + level=level + 1) + else: + return value + except TypeError, e: + # Class objects are tricky since they may define something like + # __iter__ defined but it isn't callable as list(). + return unicode(value) + + +def dumps(value, default=to_primitive, **kwargs): + return json.dumps(value, default=default, **kwargs) + + +def loads(s): + return json.loads(s) + + +def load(s): + return json.load(s) + + +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append((__name__, 'dumps', TypeError, + 'loads', ValueError, 'load')) + anyjson.force_implementation(__name__) diff --git a/heat/openstack/common/log.py b/heat/openstack/common/log.py new file mode 100644 index 0000000000..e4c808fd18 --- /dev/null +++ b/heat/openstack/common/log.py @@ -0,0 +1,459 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Openstack logging handler. + +This module adds to logging functionality by adding the option to specify +a context object when calling the various log methods. If the context object +is not specified, default formatting is used. Additionally, an instance uuid +may be passed as part of the log message, which is intended to make it easier +for admins to find messages related to a specific instance. + +It also allows setting of formatting information through conf. + +""" + +import cStringIO +import inspect +import itertools +import logging +import logging.config +import logging.handlers +import os +import stat +import sys +import traceback + +from heat.openstack.common import cfg +from heat.openstack.common.gettextutils import _ +from heat.openstack.common import jsonutils +from heat.openstack.common import local +from heat.openstack.common import notifier + + +log_opts = [ + cfg.StrOpt('logging_context_format_string', + default='%(asctime)s %(levelname)s %(name)s [%(request_id)s ' + '%(user_id)s %(project_id)s] %(instance)s' + '%(message)s', + help='format string to use for log messages with context'), + cfg.StrOpt('logging_default_format_string', + default='%(asctime)s %(levelname)s %(name)s [-] %(instance)s' + '%(message)s', + help='format string to use for log messages without context'), + cfg.StrOpt('logging_debug_format_suffix', + default='from (pid=%(process)d) %(funcName)s ' + '%(pathname)s:%(lineno)d', + help='data to append to log format when level is DEBUG'), + cfg.StrOpt('logging_exception_prefix', + default='%(asctime)s TRACE %(name)s %(instance)s', + help='prefix each line of exception output with this format'), + cfg.ListOpt('default_log_levels', + default=[ + 'amqplib=WARN', + 'sqlalchemy=WARN', + 'boto=WARN', + 'suds=INFO', + 'keystone=INFO', + 'eventlet.wsgi.server=WARN' + ], + help='list of logger=LEVEL pairs'), + cfg.BoolOpt('publish_errors', + default=False, + help='publish error events'), + + # NOTE(mikal): there are two options here because sometimes we are handed + # a full instance (and could include more information), and other times we + # are just handed a UUID for the instance. + cfg.StrOpt('instance_format', + default='[instance: %(uuid)s] ', + help='If an instance is passed with the log message, format ' + 'it like this'), + cfg.StrOpt('instance_uuid_format', + default='[instance: %(uuid)s] ', + help='If an instance UUID is passed with the log message, ' + 'format it like this'), + ] + + +generic_log_opts = [ + cfg.StrOpt('logdir', + default=None, + help='Log output to a per-service log file in named directory'), + cfg.StrOpt('logfile', + default=None, + help='Log output to a named file'), + cfg.BoolOpt('use_stderr', + default=True, + help='Log output to standard error'), + cfg.StrOpt('logfile_mode', + default='0644', + help='Default file mode used when creating log files'), + ] + + +CONF = cfg.CONF +CONF.register_opts(generic_log_opts) +CONF.register_opts(log_opts) + +# our new audit level +# NOTE(jkoelker) Since we synthesized an audit level, make the logging +# module aware of it so it acts like other levels. +logging.AUDIT = logging.INFO + 1 +logging.addLevelName(logging.AUDIT, 'AUDIT') + + +try: + NullHandler = logging.NullHandler +except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7 + class NullHandler(logging.Handler): + def handle(self, record): + pass + + def emit(self, record): + pass + + def createLock(self): + self.lock = None + + +def _dictify_context(context): + if context is None: + return None + if not isinstance(context, dict) and getattr(context, 'to_dict', None): + context = context.to_dict() + return context + + +def _get_binary_name(): + return os.path.basename(inspect.stack()[-1][1]) + + +def _get_log_file_path(binary=None): + logfile = CONF.log_file or CONF.logfile + logdir = CONF.log_dir or CONF.logdir + + if logfile and not logdir: + return logfile + + if logfile and logdir: + return os.path.join(logdir, logfile) + + if logdir: + binary = binary or _get_binary_name() + return '%s.log' % (os.path.join(logdir, binary),) + + +class ContextAdapter(logging.LoggerAdapter): + warn = logging.LoggerAdapter.warning + + def __init__(self, logger, project_name, version_string): + self.logger = logger + self.project = project_name + self.version = version_string + + def audit(self, msg, *args, **kwargs): + self.log(logging.AUDIT, msg, *args, **kwargs) + + def process(self, msg, kwargs): + if 'extra' not in kwargs: + kwargs['extra'] = {} + extra = kwargs['extra'] + + context = kwargs.pop('context', None) + if not context: + context = getattr(local.store, 'context', None) + if context: + extra.update(_dictify_context(context)) + + instance = kwargs.pop('instance', None) + instance_extra = '' + if instance: + instance_extra = CONF.instance_format % instance + else: + instance_uuid = kwargs.pop('instance_uuid', None) + if instance_uuid: + instance_extra = (CONF.instance_uuid_format + % {'uuid': instance_uuid}) + extra.update({'instance': instance_extra}) + + extra.update({"project": self.project}) + extra.update({"version": self.version}) + extra['extra'] = extra.copy() + return msg, kwargs + + +class JSONFormatter(logging.Formatter): + def __init__(self, fmt=None, datefmt=None): + # NOTE(jkoelker) we ignore the fmt argument, but its still there + # since logging.config.fileConfig passes it. + self.datefmt = datefmt + + def formatException(self, ei, strip_newlines=True): + lines = traceback.format_exception(*ei) + if strip_newlines: + lines = [itertools.ifilter(lambda x: x, + line.rstrip().splitlines()) + for line in lines] + lines = list(itertools.chain(*lines)) + return lines + + def format(self, record): + message = {'message': record.getMessage(), + 'asctime': self.formatTime(record, self.datefmt), + 'name': record.name, + 'msg': record.msg, + 'args': record.args, + 'levelname': record.levelname, + 'levelno': record.levelno, + 'pathname': record.pathname, + 'filename': record.filename, + 'module': record.module, + 'lineno': record.lineno, + 'funcname': record.funcName, + 'created': record.created, + 'msecs': record.msecs, + 'relative_created': record.relativeCreated, + 'thread': record.thread, + 'thread_name': record.threadName, + 'process_name': record.processName, + 'process': record.process, + 'traceback': None} + + if hasattr(record, 'extra'): + message['extra'] = record.extra + + if record.exc_info: + message['traceback'] = self.formatException(record.exc_info) + + return jsonutils.dumps(message) + + +class PublishErrorsHandler(logging.Handler): + def emit(self, record): + if 'list_notifier_drivers' in CONF: + if ('heat.openstack.common.notifier.log_notifier' in + CONF.list_notifier_drivers): + return + notifier.api.notify(None, 'error.publisher', + 'error_notification', + notifier.api.ERROR, + dict(error=record.msg)) + + +def handle_exception(type, value, tb): + extra = {} + if CONF.verbose: + extra['exc_info'] = (type, value, tb) + getLogger().critical(str(value), **extra) + + +def setup(product_name): + """Setup logging.""" + sys.excepthook = handle_exception + + if CONF.log_config: + try: + logging.config.fileConfig(CONF.log_config) + except Exception: + traceback.print_exc() + raise + else: + _setup_logging_from_conf(product_name) + + +def _find_facility_from_conf(): + facility_names = logging.handlers.SysLogHandler.facility_names + facility = getattr(logging.handlers.SysLogHandler, + CONF.syslog_log_facility, + None) + + if facility is None and CONF.syslog_log_facility in facility_names: + facility = facility_names.get(CONF.syslog_log_facility) + + if facility is None: + valid_facilities = facility_names.keys() + consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON', + 'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS', + 'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP', + 'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3', + 'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7'] + valid_facilities.extend(consts) + raise TypeError(_('syslog facility must be one of: %s') % + ', '.join("'%s'" % fac + for fac in valid_facilities)) + + return facility + + +def _setup_logging_from_conf(product_name): + log_root = getLogger(product_name).logger + for handler in log_root.handlers: + log_root.removeHandler(handler) + + if CONF.use_syslog: + facility = _find_facility_from_conf() + syslog = logging.handlers.SysLogHandler(address='/dev/log', + facility=facility) + log_root.addHandler(syslog) + + logpath = _get_log_file_path() + if logpath: + filelog = logging.handlers.WatchedFileHandler(logpath) + log_root.addHandler(filelog) + + mode = int(CONF.logfile_mode, 8) + st = os.stat(logpath) + if st.st_mode != (stat.S_IFREG | mode): + os.chmod(logpath, mode) + + if CONF.use_stderr: + streamlog = ColorHandler() + log_root.addHandler(streamlog) + + elif not CONF.log_file: + # pass sys.stdout as a positional argument + # python2.6 calls the argument strm, in 2.7 it's stream + streamlog = logging.StreamHandler(sys.stdout) + log_root.addHandler(streamlog) + + if CONF.publish_errors: + log_root.addHandler(PublishErrorsHandler(logging.ERROR)) + + for handler in log_root.handlers: + datefmt = CONF.log_date_format + if CONF.log_format: + handler.setFormatter(logging.Formatter(fmt=CONF.log_format, + datefmt=datefmt)) + handler.setFormatter(LegacyFormatter(datefmt=datefmt)) + + if CONF.verbose or CONF.debug: + log_root.setLevel(logging.DEBUG) + else: + log_root.setLevel(logging.INFO) + + level = logging.NOTSET + for pair in CONF.default_log_levels: + mod, _sep, level_name = pair.partition('=') + level = logging.getLevelName(level_name) + logger = logging.getLogger(mod) + logger.setLevel(level) + for handler in log_root.handlers: + logger.addHandler(handler) + + # NOTE(jkoelker) Clear the handlers for the root logger that was setup + # by basicConfig in nova/__init__.py and install the + # NullHandler. + root = logging.getLogger() + for handler in root.handlers: + root.removeHandler(handler) + handler = NullHandler() + handler.setFormatter(logging.Formatter()) + root.addHandler(handler) + + +_loggers = {} + + +def getLogger(name='unknown', version='unknown'): + if name not in _loggers: + _loggers[name] = ContextAdapter(logging.getLogger(name), + name, + version) + return _loggers[name] + + +class WritableLogger(object): + """A thin wrapper that responds to `write` and logs.""" + + def __init__(self, logger, level=logging.INFO): + self.logger = logger + self.level = level + + def write(self, msg): + self.logger.log(self.level, msg) + + +class LegacyFormatter(logging.Formatter): + """A context.RequestContext aware formatter configured through flags. + + The flags used to set format strings are: logging_context_format_string + and logging_default_format_string. You can also specify + logging_debug_format_suffix to append extra formatting if the log level is + debug. + + For information about what variables are available for the formatter see: + http://docs.python.org/library/logging.html#formatter + + """ + + def format(self, record): + """Uses contextstring if request_id is set, otherwise default.""" + if 'instance' not in record.__dict__: + record.__dict__['instance'] = '' + + if record.__dict__.get('request_id', None): + self._fmt = CONF.logging_context_format_string + else: + self._fmt = CONF.logging_default_format_string + + if (record.levelno == logging.DEBUG and + CONF.logging_debug_format_suffix): + self._fmt += " " + CONF.logging_debug_format_suffix + + # Cache this on the record, Logger will respect our formated copy + if record.exc_info: + record.exc_text = self.formatException(record.exc_info, record) + return logging.Formatter.format(self, record) + + def formatException(self, exc_info, record=None): + """Format exception output with CONF.logging_exception_prefix.""" + if not record: + return logging.Formatter.formatException(self, exc_info) + + stringbuffer = cStringIO.StringIO() + traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], + None, stringbuffer) + lines = stringbuffer.getvalue().split('\n') + stringbuffer.close() + + if CONF.logging_exception_prefix.find('%(asctime)') != -1: + record.asctime = self.formatTime(record, self.datefmt) + + formatted_lines = [] + for line in lines: + pl = CONF.logging_exception_prefix % record.__dict__ + fl = '%s%s' % (pl, line) + formatted_lines.append(fl) + return '\n'.join(formatted_lines) + + +class ColorHandler(logging.StreamHandler): + LEVEL_COLORS = { + logging.DEBUG: '\033[00;32m', # GREEN + logging.INFO: '\033[00;36m', # CYAN + logging.AUDIT: '\033[01;36m', # BOLD CYAN + logging.WARN: '\033[01;33m', # BOLD YELLOW + logging.ERROR: '\033[01;31m', # BOLD RED + logging.CRITICAL: '\033[01;31m', # BOLD RED + } + + def format(self, record): + record.color = self.LEVEL_COLORS[record.levelno] + return logging.StreamHandler.format(self, record) diff --git a/heat/openstack/common/notifier/__init__.py b/heat/openstack/common/notifier/__init__.py new file mode 100644 index 0000000000..482d54e4fd --- /dev/null +++ b/heat/openstack/common/notifier/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/heat/openstack/common/notifier/api.py b/heat/openstack/common/notifier/api.py new file mode 100644 index 0000000000..b18fff770f --- /dev/null +++ b/heat/openstack/common/notifier/api.py @@ -0,0 +1,142 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import inspect +import uuid + +from heat.openstack.common import cfg +from heat.openstack.common import context +from heat.openstack.common.gettextutils import _ +from heat.openstack.common import importutils +from heat.openstack.common import jsonutils +from heat.openstack.common import log as logging +from heat.openstack.common import timeutils + + +LOG = logging.getLogger(__name__) + +notifier_opts = [ + cfg.StrOpt('notification_driver', + default='heat.openstack.common.notifier.no_op_notifier', + help='Default driver for sending notifications'), + cfg.StrOpt('default_notification_level', + default='INFO', + help='Default notification level for outgoing notifications'), + cfg.StrOpt('default_publisher_id', + default='$host', + help='Default publisher_id for outgoing notifications'), + ] + +CONF = cfg.CONF +CONF.register_opts(notifier_opts) + +WARN = 'WARN' +INFO = 'INFO' +ERROR = 'ERROR' +CRITICAL = 'CRITICAL' +DEBUG = 'DEBUG' + +log_levels = (DEBUG, WARN, INFO, ERROR, CRITICAL) + + +class BadPriorityException(Exception): + pass + + +def notify_decorator(name, fn): + """ decorator for notify which is used from utils.monkey_patch() + + :param name: name of the function + :param function: - object of the function + :returns: function -- decorated function + + """ + def wrapped_func(*args, **kwarg): + body = {} + body['args'] = [] + body['kwarg'] = {} + for arg in args: + body['args'].append(arg) + for key in kwarg: + body['kwarg'][key] = kwarg[key] + + ctxt = context.get_context_from_function_and_args(fn, args, kwarg) + notify(ctxt, + CONF.default_publisher_id, + name, + CONF.default_notification_level, + body) + return fn(*args, **kwarg) + return wrapped_func + + +def publisher_id(service, host=None): + if not host: + host = CONF.host + return "%s.%s" % (service, host) + + +def notify(context, publisher_id, event_type, priority, payload): + """Sends a notification using the specified driver + + :param publisher_id: the source worker_type.host of the message + :param event_type: the literal type of event (ex. Instance Creation) + :param priority: patterned after the enumeration of Python logging + levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL) + :param payload: A python dictionary of attributes + + Outgoing message format includes the above parameters, and appends the + following: + + message_id + a UUID representing the id for this notification + + timestamp + the GMT timestamp the notification was sent at + + The composite message will be constructed as a dictionary of the above + attributes, which will then be sent via the transport mechanism defined + by the driver. + + Message example:: + + {'message_id': str(uuid.uuid4()), + 'publisher_id': 'compute.host1', + 'timestamp': timeutils.utcnow(), + 'priority': 'WARN', + 'event_type': 'compute.create_instance', + 'payload': {'instance_id': 12, ... }} + + """ + if priority not in log_levels: + raise BadPriorityException( + _('%s not in valid priorities') % priority) + + # Ensure everything is JSON serializable. + payload = jsonutils.to_primitive(payload, convert_instances=True) + + driver = importutils.import_module(CONF.notification_driver) + msg = dict(message_id=str(uuid.uuid4()), + publisher_id=publisher_id, + event_type=event_type, + priority=priority, + payload=payload, + timestamp=str(timeutils.utcnow())) + try: + driver.notify(context, msg) + except Exception, e: + LOG.exception(_("Problem '%(e)s' attempting to " + "send to notification system. Payload=%(payload)s") % + locals()) diff --git a/heat/openstack/common/notifier/list_notifier.py b/heat/openstack/common/notifier/list_notifier.py new file mode 100644 index 0000000000..0a0586eaf9 --- /dev/null +++ b/heat/openstack/common/notifier/list_notifier.py @@ -0,0 +1,117 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from heat.openstack.common import cfg +from heat.openstack.common.gettextutils import _ +from heat.openstack.common import importutils +from heat.openstack.common import log as logging + + +list_notifier_drivers_opt = cfg.MultiStrOpt('list_notifier_drivers', + default=['heat.openstack.common.notifier.no_op_notifier'], + help='List of drivers to send notifications') + +CONF = cfg.CONF +CONF.register_opt(list_notifier_drivers_opt) + +LOG = logging.getLogger(__name__) + +drivers = None + + +class ImportFailureNotifier(object): + """Noisily re-raises some exception over-and-over when notify is called.""" + + def __init__(self, exception): + self.exception = exception + + def notify(self, context, message): + raise self.exception + + +def _get_drivers(): + """Instantiates and returns drivers based on the flag values.""" + global drivers + if drivers is None: + drivers = [] + for notification_driver in CONF.list_notifier_drivers: + try: + drivers.append(importutils.import_module(notification_driver)) + except ImportError as e: + drivers.append(ImportFailureNotifier(e)) + return drivers + + +def add_driver(notification_driver): + """Add a notification driver at runtime.""" + # Make sure the driver list is initialized. + _get_drivers() + if isinstance(notification_driver, basestring): + # Load and add + try: + drivers.append(importutils.import_module(notification_driver)) + except ImportError as e: + drivers.append(ImportFailureNotifier(e)) + else: + # Driver is already loaded; just add the object. + drivers.append(notification_driver) + + +def _object_name(obj): + name = [] + if hasattr(obj, '__module__'): + name.append(obj.__module__) + if hasattr(obj, '__name__'): + name.append(obj.__name__) + else: + name.append(obj.__class__.__name__) + return '.'.join(name) + + +def remove_driver(notification_driver): + """Remove a notification driver at runtime.""" + # Make sure the driver list is initialized. + _get_drivers() + removed = False + if notification_driver in drivers: + # We're removing an object. Easy. + drivers.remove(notification_driver) + removed = True + else: + # We're removing a driver by name. Search for it. + for driver in drivers: + if _object_name(driver) == notification_driver: + drivers.remove(driver) + removed = True + + if not removed: + raise ValueError("Cannot remove; %s is not in list" % + notification_driver) + + +def notify(context, message): + """Passes notification to multiple notifiers in a list.""" + for driver in _get_drivers(): + try: + driver.notify(context, message) + except Exception as e: + LOG.exception(_("Problem '%(e)s' attempting to send to " + "notification driver %(driver)s."), locals()) + + +def _reset_drivers(): + """Used by unit tests to reset the drivers.""" + global drivers + drivers = None diff --git a/heat/openstack/common/notifier/log_notifier.py b/heat/openstack/common/notifier/log_notifier.py new file mode 100644 index 0000000000..0d6090d10b --- /dev/null +++ b/heat/openstack/common/notifier/log_notifier.py @@ -0,0 +1,35 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from heat.openstack.common import cfg +from heat.openstack.common import jsonutils +from heat.openstack.common import log as logging + + +CONF = cfg.CONF + + +def notify(_context, message): + """Notifies the recipient of the desired event given the model. + Log notifications using openstack's default logging system""" + + priority = message.get('priority', + CONF.default_notification_level) + priority = priority.lower() + logger = logging.getLogger( + 'heat.openstack.common.notification.%s' % + message['event_type']) + getattr(logger, priority)(jsonutils.dumps(message)) diff --git a/heat/openstack/common/notifier/no_op_notifier.py b/heat/openstack/common/notifier/no_op_notifier.py new file mode 100644 index 0000000000..ee1ddbdcac --- /dev/null +++ b/heat/openstack/common/notifier/no_op_notifier.py @@ -0,0 +1,19 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def notify(_context, message): + """Notifies the recipient of the desired event given the model""" + pass diff --git a/heat/openstack/common/notifier/rabbit_notifier.py b/heat/openstack/common/notifier/rabbit_notifier.py new file mode 100644 index 0000000000..64b00c006b --- /dev/null +++ b/heat/openstack/common/notifier/rabbit_notifier.py @@ -0,0 +1,46 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from heat.openstack.common import cfg +from heat.openstack.common import context as req_context +from heat.openstack.common.gettextutils import _ +from heat.openstack.common import log as logging +from heat.openstack.common import rpc + +LOG = logging.getLogger(__name__) + +notification_topic_opt = cfg.ListOpt('notification_topics', + default=['notifications', ], + help='AMQP topic used for openstack notifications') + +CONF = cfg.CONF +CONF.register_opt(notification_topic_opt) + + +def notify(context, message): + """Sends a notification to the RabbitMQ""" + if not context: + context = req_context.get_admin_context() + priority = message.get('priority', + CONF.default_notification_level) + priority = priority.lower() + for topic in CONF.notification_topics: + topic = '%s.%s' % (topic, priority) + try: + rpc.notify(context, topic, message) + except Exception, e: + LOG.exception(_("Could not send notification to %(topic)s. " + "Payload=%(message)s"), locals()) diff --git a/heat/openstack/common/notifier/test_notifier.py b/heat/openstack/common/notifier/test_notifier.py new file mode 100644 index 0000000000..5e348803dc --- /dev/null +++ b/heat/openstack/common/notifier/test_notifier.py @@ -0,0 +1,22 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +NOTIFICATIONS = [] + + +def notify(_context, message): + """Test notifier, stores notifications in memory for unittests.""" + NOTIFICATIONS.append(message) diff --git a/heat/rpc/__init__.py b/heat/rpc/__init__.py index 4ae29bb087..b7cb7876c8 100644 --- a/heat/rpc/__init__.py +++ b/heat/rpc/__init__.py @@ -17,10 +17,11 @@ # License for the specific language governing permissions and limitations # under the License. -import logging from heat.openstack.common import cfg from heat.openstack.common import importutils +from heat.openstack.common import log as logging + LOG = logging.getLogger(__name__) diff --git a/heat/rpc/amqp.py b/heat/rpc/amqp.py index 8f02f9471e..8d7989355f 100644 --- a/heat/rpc/amqp.py +++ b/heat/rpc/amqp.py @@ -26,11 +26,12 @@ AMQP, but is deprecated and predates this code. """ import inspect -import logging import sys import traceback import uuid +from heat.openstack.common import log as logging + from eventlet import greenpool from eventlet import pools diff --git a/heat/rpc/common.py b/heat/rpc/common.py index 28d182475b..f0792c807c 100644 --- a/heat/rpc/common.py +++ b/heat/rpc/common.py @@ -18,13 +18,12 @@ # under the License. import copy -import logging from heat.openstack.common import cfg from heat.openstack.common import exception +from heat.openstack.common import log as logging from heat.common import config - LOG = logging.getLogger(__name__) diff --git a/heat/rpc/impl_qpid.py b/heat/rpc/impl_qpid.py index 7602cf1d48..879e59c12b 100644 --- a/heat/rpc/impl_qpid.py +++ b/heat/rpc/impl_qpid.py @@ -19,7 +19,6 @@ import itertools import time import uuid import json -import logging import eventlet import greenlet @@ -31,6 +30,8 @@ from heat.openstack.common import cfg from heat.rpc import amqp as rpc_amqp from heat.rpc import common as rpc_common +from heat.openstack.common import log as logging + LOG = logging.getLogger(__name__) diff --git a/heat/service.py b/heat/service.py index 9a118ff67c..68472b8b94 100644 --- a/heat/service.py +++ b/heat/service.py @@ -23,11 +23,11 @@ import inspect import os import eventlet -import logging import greenlet from heat.openstack.common import cfg from heat.openstack.common import importutils +from heat.openstack.common import log as logging from heat.common import utils as heat_utils from heat.common import exception diff --git a/heat/utils.py b/heat/utils.py index 71770f7769..528b40dd43 100644 --- a/heat/utils.py +++ b/heat/utils.py @@ -20,11 +20,13 @@ import sys import base64 from lxml import etree import re -import logging from glance import client as glance_client from heat.common import exception +from heat.openstack.common import log as logging + +LOG = logging.getLogger(__name__) SUCCESS = 0 FAILURE = 1 @@ -39,14 +41,14 @@ def catch_error(action): ret = func(*arguments, **kwargs) return SUCCESS if ret is None else ret except exception.NotAuthorized: - logging.error("Not authorized to make this request. Check " + + LOG.error("Not authorized to make this request. Check " + "your credentials (OS_USERNAME, OS_PASSWORD, " + "OS_TENANT_NAME, OS_AUTH_URL and OS_AUTH_STRATEGY).") return FAILURE except exception.ClientConfigurationError: raise except exception.KeystoneError, e: - logging.error("Keystone did not finish the authentication and " + LOG.error("Keystone did not finish the authentication and " "returned the following message:\n\n%s" % e.message) return FAILURE @@ -54,10 +56,10 @@ def catch_error(action): options = arguments[0] if options.debug: raise - logging.error("Failed to %s. Got error:" % action) + LOG.error("Failed to %s. Got error:" % action) pieces = unicode(e).split('\n') for piece in pieces: - logging.error(piece) + LOG.error(piece) return FAILURE return wrapper diff --git a/openstack-common.conf b/openstack-common.conf index d61ff0b602..7a14d218eb 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,7 +1,7 @@ [DEFAULT] # The list of modules to copy from openstack-common -modules=gettextutils,cfg,local,iniparser,utils,exception,timeutils,importutils,setup +modules=gettextutils,cfg,local,iniparser,utils,exception,timeutils,importutils,setup,log,jsonutils,notifier # The base module to hold the copy of openstack.common base=heat