# -*- coding: utf-8 -*- # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. # Copyright (C) 2013 Rackspace Hosting All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import collections import contextlib import copy import inspect import logging import threading import time import types import weakref import threading2 from distutils import version from taskflow.openstack.common import uuidutils from taskflow import states TASK_FACTORY_ATTRIBUTE = '_TaskFlow_task_factory' LOG = logging.getLogger(__name__) def await(check_functor, timeout=None): if timeout is not None: end_time = time.time() + max(0, timeout) else: end_time = None # Use the same/similar scheme that the python condition class uses. delay = 0.0005 while not check_functor(): time.sleep(delay) if end_time is not None: remaining = end_time - time.time() if remaining <= 0: return False delay = min(delay * 2, remaining, 0.05) else: delay = min(delay * 2, 0.05) return True def get_callable_name(function): """Generate a name from callable Tries to do the best to guess fully qualified callable name. """ im_class = getattr(function, 'im_class', None) if im_class is not None: if im_class is type: # this is bound class method im_class = function.im_self parts = (im_class.__module__, im_class.__name__, function.__name__) elif isinstance(function, types.FunctionType): parts = (function.__module__, function.__name__) else: im_class = type(function) if im_class is type: im_class = function parts = (im_class.__module__, im_class.__name__) return '.'.join(parts) def is_bound_method(method): return getattr(method, 'im_self', None) is not None def get_required_callable_args(function): """Get names of argument required by callable""" if isinstance(function, type): bound = True function = function.__init__ elif isinstance(function, (types.FunctionType, types.MethodType)): bound = is_bound_method(function) function = getattr(function, '__wrapped__', function) else: function = function.__call__ bound = is_bound_method(function) argspec = inspect.getargspec(function) f_args = argspec.args if argspec.defaults: f_args = f_args[:-len(argspec.defaults)] if bound: f_args = f_args[1:] return f_args def get_task_version(task): """Gets a tasks *string* version, whether it is a task object/function.""" task_version = getattr(task, 'version') if isinstance(task_version, (list, tuple)): task_version = '.'.join(str(item) for item in task_version) if task_version is not None and not isinstance(task_version, basestring): task_version = str(task_version) return task_version def is_version_compatible(version_1, version_2): """Checks for major version compatibility of two *string" versions.""" try: version_1_tmp = version.StrictVersion(version_1) version_2_tmp = version.StrictVersion(version_2) except ValueError: version_1_tmp = version.LooseVersion(version_1) version_2_tmp = version.LooseVersion(version_2) version_1 = version_1_tmp version_2 = version_2_tmp if version_1 == version_2 or version_1.version[0] == version_2.version[0]: return True return False class MultiLock(object): """A class which can attempt to obtain many locks at once and release said locks when exiting. Useful as a context manager around many locks (instead of having to nest said individual context managers). """ def __init__(self, locks): assert len(locks) > 0, "Zero locks requested" self._locks = locks self._locked = [False] * len(locks) def __enter__(self): def is_locked(lock): # NOTE(harlowja): the threading2 lock doesn't seem to have this # attribute, so thats why we are checking it existing first. if hasattr(lock, 'locked'): return lock.locked() return False for i in xrange(0, len(self._locked)): if self._locked[i] or is_locked(self._locks[i]): raise threading.ThreadError("Lock %s not previously released" % (i + 1)) self._locked[i] = False for (i, lock) in enumerate(self._locks): self._locked[i] = lock.acquire() def __exit__(self, type, value, traceback): for (i, locked) in enumerate(self._locked): try: if locked: self._locks[i].release() self._locked[i] = False except threading.ThreadError: LOG.exception("Unable to release lock %s", i + 1) class CountDownLatch(object): """Similar in concept to the java count down latch.""" def __init__(self, count=0): self.count = count self.lock = threading.Condition() def countDown(self): with self.lock: self.count -= 1 if self.count <= 0: self.lock.notifyAll() def await(self, timeout=None): end_time = None if timeout is not None: timeout = max(0, timeout) end_time = time.time() + timeout time_up = False with self.lock: while True: # Stop waiting on these 2 conditions. if time_up or self.count <= 0: break # Was this a spurious wakeup or did we really end?? self.lock.wait(timeout=timeout) if end_time is not None: if time.time() >= end_time: time_up = True else: # Reduce the timeout so that we don't wait extra time # over what we initially were requested to. timeout = end_time - time.time() return self.count <= 0 class LastFedIter(object): """An iterator which yields back the first item and then yields back results from the provided iterator. """ def __init__(self, first, rest_itr): self.first = first self.rest_itr = rest_itr def __iter__(self): yield self.first for i in self.rest_itr: yield i class ThreadGroupExecutor(object): """A simple thread executor that spins up new threads (or greenthreads) for each task to be completed (no pool limit is enforced). TODO(harlowja): Likely if we use the more advanced executors that come with the concurrent.futures library we can just get rid of this. """ def __init__(self, daemonize=True): self._threads = [] self._group = threading2.ThreadGroup() self._daemonize = daemonize def submit(self, fn, *args, **kwargs): t = threading2.Thread(target=fn, group=self._group, args=args, kwargs=kwargs) t.daemon = self._daemonize self._threads.append(t) t.start() def await_termination(self, timeout=None): if not self._threads: return return self._group.join(timeout) class FlowFailure(object): """When a task failure occurs the following object will be given to revert and can be used to interrogate what caused the failure. """ def __init__(self, runner, flow): self.runner = runner self.flow = flow @property def exc_info(self): return self.runner.exc_info @property def exc(self): return self.runner.exc_info[1] class Runner(object): """A helper class that wraps a task and can find the needed inputs for the task to run, as well as providing a uuid and other useful functionality for users of the task. """ def __init__(self, task, uuid=None): assert isinstance(task, collections.Callable) task_factory = getattr(task, TASK_FACTORY_ATTRIBUTE, None) if task_factory: self.task = task_factory(task) else: self.task = task self.providers = {} self.result = None if not uuid: self._id = uuidutils.generate_uuid() else: self._id = str(uuid) self.exc_info = (None, None, None) @property def uuid(self): return str(self._id) @property def requires(self): return self.task.requires @property def provides(self): return self.task.provides @property def optional(self): return self.task.optional @property def runs_before(self): return [] @property def version(self): return get_task_version(self.task) @property def name(self): if hasattr(self.task, 'name'): return self.task.name return '?' def reset(self): self.result = None self.exc_info = (None, None, None) def __str__(self): lines = ["Runner: %s" % (self.name)] lines.append("%s" % (self.uuid)) lines.append("%s" % (self.version)) return "; ".join(lines) def __call__(self, *args, **kwargs): # Find all of our inputs first. kwargs = dict(kwargs) for (k, who_made) in self.providers.iteritems(): if k in kwargs: continue try: kwargs[k] = who_made.result[k] except (TypeError, KeyError): pass optional_keys = self.optional optional_keys = optional_keys - set(kwargs.keys()) for k in optional_keys: for who_ran in self.runs_before: matched = False if k in who_ran.provides: try: kwargs[k] = who_ran.result[k] matched = True except (TypeError, KeyError): pass if matched: break # Ensure all required keys are either existent or set to none. for k in self.requires: if k not in kwargs: kwargs[k] = None # And now finally run. self.result = self.task(*args, **kwargs) return self.result class AOTRunner(Runner): """A runner that knows who runs before this runner ahead of time from a known list of previous runners. """ def __init__(self, task): super(AOTRunner, self).__init__(task) self._runs_before = [] @property def runs_before(self): return self._runs_before @runs_before.setter def runs_before(self, runs_before): self._runs_before = list(runs_before) class TransitionNotifier(object): """A utility helper class that can be used to subscribe to notifications of events occuring as well as allow a entity to post said notifications to subscribers. """ RESERVED_KEYS = ('details',) ANY = '*' def __init__(self): self._listeners = collections.defaultdict(list) def reset(self): self._listeners = collections.defaultdict(list) def notify(self, state, details): listeners = list(self._listeners.get(self.ANY, [])) for i in self._listeners[state]: if i not in listeners: listeners.append(i) if not listeners: return for (callback, args, kwargs) in listeners: if args is None: args = [] if kwargs is None: kwargs = {} kwargs['details'] = details try: callback(state, *args, **kwargs) except Exception: LOG.exception(("Failure calling callback %s to notify about" " state transition %s"), callback, state) def register(self, state, callback, args=None, kwargs=None): assert isinstance(callback, collections.Callable) for i, (cb, args, kwargs) in enumerate(self._listeners.get(state, [])): if cb is callback: raise ValueError("Callback %s already registered" % (callback)) if kwargs: for k in self.RESERVED_KEYS: if k in kwargs: raise KeyError(("Reserved key '%s' not allowed in " "kwargs") % k) kwargs = copy.copy(kwargs) if args: args = copy.copy(args) self._listeners[state].append((callback, args, kwargs)) def deregister(self, state, callback): if state not in self._listeners: return for i, (cb, args, kwargs) in enumerate(self._listeners[state]): if cb is callback: self._listeners[state].pop(i) break class Rollback(object): """A helper functor object that on being called will call the underlying runners tasks revert method (if said method exists) and do the appropriate notification to signal to others that the reverting is underway. """ def __init__(self, context, runner, flow, notifier): self.runner = runner self.context = context self.notifier = notifier # Use weak references to give the GC a break. self.flow = weakref.proxy(flow) def __str__(self): return "Rollback: %s" % (self.runner) def _fire_notify(self, has_reverted): if self.notifier: if has_reverted: state = states.REVERTED else: state = states.REVERTING self.notifier.notify(state, details={ 'context': self.context, 'flow': self.flow, 'runner': self.runner, }) def __call__(self, cause): self._fire_notify(False) task = self.runner.task if ((hasattr(task, "revert") and isinstance(task.revert, collections.Callable))): task.revert(self.context, self.runner.result, cause) self._fire_notify(True) class RollbackAccumulator(object): """A utility class that can help in organizing 'undo' like code so that said code be rolled back on failure (automatically or manually) by activating rollback callables that were inserted during said codes progression. """ def __init__(self): self._rollbacks = [] def add(self, *callables): self._rollbacks.extend(callables) def reset(self): self._rollbacks = [] def __len__(self): return len(self._rollbacks) def __enter__(self): return self def rollback(self, cause): LOG.warn("Activating %s rollbacks due to %s.", len(self), cause) for (i, f) in enumerate(reversed(self._rollbacks)): LOG.debug("Calling rollback %s: %s", i + 1, f) try: f(cause) except Exception: LOG.exception(("Failed rolling back %s: %s due " "to inner exception."), i + 1, f) def __exit__(self, type, value, tb): if any((value, type, tb)): self.rollback(value) class ReaderWriterLock(object): """A simple reader-writer lock. Several readers can hold the lock simultaneously, and only one writer. Write locks have priority over reads to prevent write starvation. Public domain @ http://majid.info/blog/a-reader-writer-lock-for-python/ """ def __init__(self): self.rwlock = 0 self.writers_waiting = 0 self.monitor = threading.Lock() self.readers_ok = threading.Condition(self.monitor) self.writers_ok = threading.Condition(self.monitor) @contextlib.contextmanager def acquire(self, read=True): """Acquire a read or write lock in a context manager.""" try: if read: self.acquire_read() else: self.acquire_write() yield self finally: self.release() def acquire_read(self): """Acquire a read lock. Several threads can hold this typeof lock. It is exclusive with write locks. """ self.monitor.acquire() while self.rwlock < 0 or self.writers_waiting: self.readers_ok.wait() self.rwlock += 1 self.monitor.release() def acquire_write(self): """Acquire a write lock. Only one thread can hold this lock, and only when no read locks are also held. """ self.monitor.acquire() while self.rwlock != 0: self.writers_waiting += 1 self.writers_ok.wait() self.writers_waiting -= 1 self.rwlock = -1 self.monitor.release() def release(self): """Release a lock, whether read or write.""" self.monitor.acquire() if self.rwlock < 0: self.rwlock = 0 else: self.rwlock -= 1 wake_writers = self.writers_waiting and self.rwlock == 0 wake_readers = self.writers_waiting == 0 self.monitor.release() if wake_writers: self.writers_ok.acquire() self.writers_ok.notify() self.writers_ok.release() elif wake_readers: self.readers_ok.acquire() self.readers_ok.notifyAll() self.readers_ok.release()