diff --git a/taskflow/atom.py b/taskflow/atom.py index b3b7ea9c6..2cd665ac9 100644 --- a/taskflow/atom.py +++ b/taskflow/atom.py @@ -15,16 +15,12 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - import six from taskflow import exceptions from taskflow.utils import misc from taskflow.utils import reflection -LOG = logging.getLogger(__name__) - def _save_as_to_mapping(save_as): """Convert save_as to mapping name => index. diff --git a/taskflow/conductors/single_threaded.py b/taskflow/conductors/single_threaded.py index 173d2cec8..e39f4c492 100644 --- a/taskflow/conductors/single_threaded.py +++ b/taskflow/conductors/single_threaded.py @@ -12,13 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - import six from taskflow.conductors import base from taskflow import exceptions as excp from taskflow.listeners import logging as logging_listener +from taskflow import logging from taskflow.types import timing as tt from taskflow.utils import async_utils from taskflow.utils import lock_utils diff --git a/taskflow/engines/action_engine/actions/retry.py b/taskflow/engines/action_engine/actions/retry.py index 3262a79f7..5f6eb90c0 100644 --- a/taskflow/engines/action_engine/actions/retry.py +++ b/taskflow/engines/action_engine/actions/retry.py @@ -14,9 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - from taskflow.engines.action_engine import executor as ex +from taskflow import logging from taskflow import retry as retry_atom from taskflow import states from taskflow.types import failure diff --git a/taskflow/engines/action_engine/actions/task.py b/taskflow/engines/action_engine/actions/task.py index 6c520f160..b6bc6e771 100644 --- a/taskflow/engines/action_engine/actions/task.py +++ b/taskflow/engines/action_engine/actions/task.py @@ -14,8 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - +from taskflow import logging from taskflow import states from taskflow import task as task_atom from taskflow.types import failure diff --git a/taskflow/engines/action_engine/compiler.py b/taskflow/engines/action_engine/compiler.py index 47bc4a0b5..38b48db65 100644 --- a/taskflow/engines/action_engine/compiler.py +++ b/taskflow/engines/action_engine/compiler.py @@ -14,11 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import threading from taskflow import exceptions as exc from taskflow import flow +from taskflow import logging from taskflow import retry from taskflow import task from taskflow.types import graph as gr @@ -190,18 +190,18 @@ class PatternCompiler(object): % (self._root, type(self._root))) self._history.clear() # NOTE(harlowja): this one can be expensive to calculate (especially - # the cycle detection), so only do it if we know debugging is enabled + # the cycle detection), so only do it if we know BLATHER is enabled # and not under all cases. - if LOG.isEnabledFor(logging.DEBUG): - LOG.debug("Translated '%s'", self._root) - LOG.debug("Graph:") + if LOG.isEnabledFor(logging.BLATHER): + LOG.blather("Translated '%s'", self._root) + LOG.blather("Graph:") for line in graph.pformat().splitlines(): # Indent it so that it's slightly offset from the above line. - LOG.debug(" %s", line) - LOG.debug("Hierarchy:") + LOG.blather(" %s", line) + LOG.blather("Hierarchy:") for line in node.pformat().splitlines(): # Indent it so that it's slightly offset from the above line. - LOG.debug(" %s", line) + LOG.blather(" %s", line) @lock_utils.locked def compile(self): diff --git a/taskflow/engines/action_engine/runner.py b/taskflow/engines/action_engine/runner.py index 502bad186..6d6105694 100644 --- a/taskflow/engines/action_engine/runner.py +++ b/taskflow/engines/action_engine/runner.py @@ -14,8 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - +from taskflow import logging from taskflow import states as st from taskflow.types import failure from taskflow.types import fsm diff --git a/taskflow/engines/action_engine/scopes.py b/taskflow/engines/action_engine/scopes.py index f1dd49d15..6b7f9ffda 100644 --- a/taskflow/engines/action_engine/scopes.py +++ b/taskflow/engines/action_engine/scopes.py @@ -14,10 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - from taskflow import atom as atom_type from taskflow import flow as flow_type +from taskflow import logging LOG = logging.getLogger(__name__) @@ -102,18 +101,13 @@ class ScopeWalker(object): visible.append(a) else: visible.append(a.name) - if LOG.isEnabledFor(logging.DEBUG): + if LOG.isEnabledFor(logging.BLATHER): if not self._names_only: visible_names = [a.name for a in visible] else: visible_names = visible - # TODO(harlowja): we should likely use a created TRACE level - # for this kind of *very* verbose information; otherwise the - # cinder and other folks are going to complain that there - # debug logs are full of not so useful information (it is - # useful to taskflow debugging...). - LOG.debug("Scope visible to '%s' (limited by parent '%s' index" - " < %s) is: %s", self._atom, parent.item.name, - last_idx, visible_names) + LOG.blather("Scope visible to '%s' (limited by parent '%s'" + " index < %s) is: %s", self._atom, + parent.item.name, last_idx, visible_names) yield visible last = parent diff --git a/taskflow/engines/helpers.py b/taskflow/engines/helpers.py index 61a729aa3..1943aa735 100644 --- a/taskflow/engines/helpers.py +++ b/taskflow/engines/helpers.py @@ -23,12 +23,14 @@ import six import stevedore.driver from taskflow import exceptions as exc +from taskflow import logging from taskflow.persistence import backends as p_backends from taskflow.utils import deprecation from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils from taskflow.utils import reflection +LOG = logging.getLogger(__name__) # NOTE(imelnikov): this is the entrypoint namespace, not the module namespace. ENGINES_NAMESPACE = 'taskflow.engines' @@ -171,6 +173,7 @@ def load(flow, store=None, flow_detail=None, book=None, flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend) + LOG.debug('Looking for %r engine driver in %r', kind, namespace) try: mgr = stevedore.driver.DriverManager( namespace, kind, diff --git a/taskflow/engines/worker_based/dispatcher.py b/taskflow/engines/worker_based/dispatcher.py index f354ec4b3..7ea8947c4 100644 --- a/taskflow/engines/worker_based/dispatcher.py +++ b/taskflow/engines/worker_based/dispatcher.py @@ -14,12 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - from kombu import exceptions as kombu_exc import six from taskflow import exceptions as excp +from taskflow import logging LOG = logging.getLogger(__name__) diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index 092d40385..3943a6e02 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -15,7 +15,6 @@ # under the License. import functools -import logging import threading from oslo.utils import timeutils @@ -25,6 +24,7 @@ from taskflow.engines.worker_based import cache from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy from taskflow import exceptions as exc +from taskflow import logging from taskflow.types import timing as tt from taskflow.utils import async_utils from taskflow.utils import misc diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py index 3fd872167..c2bb22d28 100644 --- a/taskflow/engines/worker_based/protocol.py +++ b/taskflow/engines/worker_based/protocol.py @@ -15,7 +15,6 @@ # under the License. import abc -import logging import threading from concurrent import futures @@ -26,6 +25,7 @@ import six from taskflow.engines.action_engine import executor from taskflow import exceptions as excp +from taskflow import logging from taskflow.types import failure as ft from taskflow.types import timing as tt from taskflow.utils import lock_utils diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index 3f279a774..1c870595a 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -15,13 +15,13 @@ # under the License. import collections -import logging import socket import kombu import six from taskflow.engines.worker_based import dispatcher +from taskflow import logging from taskflow.utils import threading_utils LOG = logging.getLogger(__name__) diff --git a/taskflow/engines/worker_based/server.py b/taskflow/engines/worker_based/server.py index db61edc60..0537dc09f 100644 --- a/taskflow/engines/worker_based/server.py +++ b/taskflow/engines/worker_based/server.py @@ -15,12 +15,12 @@ # under the License. import functools -import logging import six from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy +from taskflow import logging from taskflow.types import failure as ft from taskflow.utils import misc diff --git a/taskflow/engines/worker_based/worker.py b/taskflow/engines/worker_based/worker.py index de55a8e24..18627e27c 100644 --- a/taskflow/engines/worker_based/worker.py +++ b/taskflow/engines/worker_based/worker.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import os import platform import socket @@ -25,6 +24,7 @@ from concurrent import futures from taskflow.engines.worker_based import endpoint from taskflow.engines.worker_based import server +from taskflow import logging from taskflow import task as t_task from taskflow.utils import reflection from taskflow.utils import threading_utils as tu diff --git a/taskflow/jobs/backends/__init__.py b/taskflow/jobs/backends/__init__.py index 94afa6e56..e8bd6daf2 100644 --- a/taskflow/jobs/backends/__init__.py +++ b/taskflow/jobs/backends/__init__.py @@ -15,12 +15,12 @@ # under the License. import contextlib -import logging import six from stevedore import driver from taskflow import exceptions as exc +from taskflow import logging from taskflow.utils import misc diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index ea485f8b0..d6c131d62 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -17,7 +17,6 @@ import collections import contextlib import functools -import logging import threading from concurrent import futures @@ -31,6 +30,7 @@ import six from taskflow import exceptions as excp from taskflow.jobs import job as base_job from taskflow.jobs import jobboard +from taskflow import logging from taskflow.openstack.common import uuidutils from taskflow import states from taskflow.types import timing as tt diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index f69bb87e4..47ff8d5b5 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -17,11 +17,11 @@ from __future__ import absolute_import import abc -import logging from oslo.utils import excutils import six +from taskflow import logging from taskflow import states from taskflow.types import failure from taskflow.types import notifier diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index 51bf693c5..d707e395d 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -16,10 +16,10 @@ from __future__ import absolute_import -import logging import sys from taskflow.listeners import base +from taskflow import logging from taskflow import states from taskflow.types import failure diff --git a/taskflow/listeners/timing.py b/taskflow/listeners/timing.py index 4a08256ef..f20805475 100644 --- a/taskflow/listeners/timing.py +++ b/taskflow/listeners/timing.py @@ -17,10 +17,10 @@ from __future__ import absolute_import import itertools -import logging from taskflow import exceptions as exc from taskflow.listeners import base +from taskflow import logging from taskflow import states from taskflow.types import timing as tt diff --git a/taskflow/logging.py b/taskflow/logging.py new file mode 100644 index 000000000..a735a9aea --- /dev/null +++ b/taskflow/logging.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import + +import logging +import sys + +_BASE = __name__.split(".", 1)[0] + +# Add a BLATHER level, this matches the multiprocessing utils.py module (and +# kazoo and others) that declares a similar level, this level is for +# information that is even lower level than regular DEBUG and gives out so +# much runtime information that it is only useful by low-level/certain users... +BLATHER = 5 + +# Copy over *select* attributes to make it easy to use this module. +CRITICAL = logging.CRITICAL +DEBUG = logging.DEBUG +ERROR = logging.ERROR +FATAL = logging.FATAL +NOTSET = logging.NOTSET +WARN = logging.WARN +WARNING = logging.WARNING + + +class _BlatherLoggerAdapter(logging.LoggerAdapter): + + def blather(self, msg, *args, **kwargs): + """Delegate a blather call to the underlying logger.""" + self.logger.log(BLATHER, msg, *args, **kwargs) + + def warn(self, msg, *args, **kwargs): + """Delegate a warning call to the underlying logger.""" + self.warning(msg, *args, **kwargs) + + +# TODO(harlowja): we should remove when we no longer have to support 2.6... +if sys.version_info[0:2] == (2, 6): + + class _FixedBlatherLoggerAdapter(_BlatherLoggerAdapter): + """Ensures isEnabledFor() exists on adapters that are created.""" + + def isEnabledFor(self, level): + return self.logger.isEnabledFor(level) + + _BlatherLoggerAdapter = _FixedBlatherLoggerAdapter + + # Taken from python2.7 (same in python3.4)... + class _NullHandler(logging.Handler): + """This handler does nothing. + + It's intended to be used to avoid the + "No handlers could be found for logger XXX" one-off warning. This is + important for library code, which may contain code to log events. If a + user of the library does not configure logging, the one-off warning + might be produced; to avoid this, the library developer simply needs + to instantiate a _NullHandler and add it to the top-level logger of the + library module or package. + """ + + def handle(self, record): + """Stub.""" + + def emit(self, record): + """Stub.""" + + def createLock(self): + self.lock = None + +else: + _NullHandler = logging.NullHandler + + +def getLogger(name=_BASE, extra=None): + logger = logging.getLogger(name) + if not logger.handlers: + logger.addHandler(_NullHandler()) + return _BlatherLoggerAdapter(logger, extra=extra) diff --git a/taskflow/persistence/backends/__init__.py b/taskflow/persistence/backends/__init__.py index 64b7cda10..cceb7e756 100644 --- a/taskflow/persistence/backends/__init__.py +++ b/taskflow/persistence/backends/__init__.py @@ -15,11 +15,11 @@ # under the License. import contextlib -import logging from stevedore import driver from taskflow import exceptions as exc +from taskflow import logging from taskflow.utils import misc diff --git a/taskflow/persistence/backends/impl_dir.py b/taskflow/persistence/backends/impl_dir.py index f469a1dbc..0a687473f 100644 --- a/taskflow/persistence/backends/impl_dir.py +++ b/taskflow/persistence/backends/impl_dir.py @@ -16,7 +16,6 @@ # under the License. import errno -import logging import os import shutil @@ -24,6 +23,7 @@ from oslo.serialization import jsonutils import six from taskflow import exceptions as exc +from taskflow import logging from taskflow.persistence.backends import base from taskflow.persistence import logbook from taskflow.utils import lock_utils diff --git a/taskflow/persistence/backends/impl_memory.py b/taskflow/persistence/backends/impl_memory.py index f425987cc..6c58718aa 100644 --- a/taskflow/persistence/backends/impl_memory.py +++ b/taskflow/persistence/backends/impl_memory.py @@ -15,11 +15,10 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - import six from taskflow import exceptions as exc +from taskflow import logging from taskflow.persistence.backends import base from taskflow.persistence import logbook diff --git a/taskflow/persistence/backends/impl_sqlalchemy.py b/taskflow/persistence/backends/impl_sqlalchemy.py index 4b12b782c..9241c7a4f 100644 --- a/taskflow/persistence/backends/impl_sqlalchemy.py +++ b/taskflow/persistence/backends/impl_sqlalchemy.py @@ -22,7 +22,6 @@ from __future__ import absolute_import import contextlib import copy import functools -import logging import time from oslo.utils import strutils @@ -33,6 +32,7 @@ from sqlalchemy import orm as sa_orm from sqlalchemy import pool as sa_pool from taskflow import exceptions as exc +from taskflow import logging from taskflow.persistence.backends import base from taskflow.persistence.backends.sqlalchemy import migration from taskflow.persistence.backends.sqlalchemy import models diff --git a/taskflow/persistence/backends/impl_zookeeper.py b/taskflow/persistence/backends/impl_zookeeper.py index 3cacffec6..ca801b438 100644 --- a/taskflow/persistence/backends/impl_zookeeper.py +++ b/taskflow/persistence/backends/impl_zookeeper.py @@ -15,13 +15,13 @@ # under the License. import contextlib -import logging from kazoo import exceptions as k_exc from kazoo.protocol import paths from oslo.serialization import jsonutils from taskflow import exceptions as exc +from taskflow import logging from taskflow.persistence.backends import base from taskflow.persistence import logbook from taskflow.utils import kazoo_utils as k_utils diff --git a/taskflow/persistence/logbook.py b/taskflow/persistence/logbook.py index f66f68efe..ea6de4d00 100644 --- a/taskflow/persistence/logbook.py +++ b/taskflow/persistence/logbook.py @@ -17,12 +17,12 @@ import abc import copy -import logging from oslo.utils import timeutils import six from taskflow import exceptions as exc +from taskflow import logging from taskflow.openstack.common import uuidutils from taskflow import states from taskflow.types import failure as ft diff --git a/taskflow/retry.py b/taskflow/retry.py index 6897e3b7c..5e996be8c 100644 --- a/taskflow/retry.py +++ b/taskflow/retry.py @@ -16,7 +16,6 @@ # under the License. import abc -import logging import six @@ -24,8 +23,6 @@ from taskflow import atom from taskflow import exceptions as exc from taskflow.utils import misc -LOG = logging.getLogger(__name__) - # Decision results. REVERT = "REVERT" REVERT_ALL = "REVERT_ALL" diff --git a/taskflow/storage.py b/taskflow/storage.py index c667509b4..9a234ae55 100644 --- a/taskflow/storage.py +++ b/taskflow/storage.py @@ -16,11 +16,11 @@ import abc import contextlib -import logging import six from taskflow import exceptions +from taskflow import logging from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow import retry @@ -696,20 +696,17 @@ class Storage(object): injected_args = {} mapped_args = {} for (bound_name, name) in six.iteritems(args_mapping): - # TODO(harlowja): This logging information may be to verbose - # even for DEBUG mode, let's see if we can maybe in the future - # add a TRACE mode or something else if people complain... - if LOG.isEnabledFor(logging.DEBUG): + if LOG.isEnabledFor(logging.BLATHER): if atom_name: - LOG.debug("Looking for %r <= %r for atom named: %s", - bound_name, name, atom_name) + LOG.blather("Looking for %r <= %r for atom named: %s", + bound_name, name, atom_name) else: - LOG.debug("Looking for %r <= %r", bound_name, name) + LOG.blather("Looking for %r <= %r", bound_name, name) if name in injected_args: value = injected_args[name] mapped_args[bound_name] = value - LOG.debug("Matched %r <= %r to %r (from injected values)", - bound_name, name, value) + LOG.blather("Matched %r <= %r to %r (from injected" + " values)", bound_name, name, value) else: try: possible_providers = self._reverse_mapping[name] @@ -727,8 +724,8 @@ class Storage(object): % (bound_name, name, len(possible_providers))) provider, value = _item_from_first_of(providers, name) mapped_args[bound_name] = value - LOG.debug("Matched %r <= %r to %r (from %s)", - bound_name, name, value, provider) + LOG.blather("Matched %r <= %r to %r (from %s)", + bound_name, name, value, provider) return mapped_args def set_flow_state(self, state): diff --git a/taskflow/task.py b/taskflow/task.py index 287a7d748..11b9972dc 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -19,11 +19,11 @@ import abc import collections import contextlib import copy -import logging import six from taskflow import atom +from taskflow import logging from taskflow.utils import reflection LOG = logging.getLogger(__name__) diff --git a/taskflow/test.py b/taskflow/test.py index c6a56a99c..bf2522293 100644 --- a/taskflow/test.py +++ b/taskflow/test.py @@ -14,6 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import absolute_import + import collections import logging diff --git a/taskflow/utils/lock_utils.py b/taskflow/utils/lock_utils.py index 2d65395e0..b61668b03 100644 --- a/taskflow/utils/lock_utils.py +++ b/taskflow/utils/lock_utils.py @@ -22,13 +22,13 @@ import collections import contextlib import errno -import logging import os import threading import time import six +from taskflow import logging from taskflow.utils import misc from taskflow.utils import threading_utils as tu diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 583cdd1aa..88148bd7e 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -19,7 +19,6 @@ import contextlib import datetime import errno import inspect -import logging import os import re import sys @@ -38,7 +37,6 @@ from taskflow.utils import deprecation from taskflow.utils import reflection -LOG = logging.getLogger(__name__) NUMERIC_TYPES = six.integer_types + (float,) # NOTE(imelnikov): regular expression to get scheme from URI, diff --git a/taskflow/utils/persistence_utils.py b/taskflow/utils/persistence_utils.py index dbcdac29f..340f558a6 100644 --- a/taskflow/utils/persistence_utils.py +++ b/taskflow/utils/persistence_utils.py @@ -15,10 +15,10 @@ # under the License. import contextlib -import logging from oslo.utils import timeutils +from taskflow import logging from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow.utils import misc