From d2fa28ecdc83ed2c124743df2b3425374428542b Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 18 Jan 2010 15:42:15 -0800 Subject: [PATCH] Doc improvements, removed docs for coros and proc modules. --- AUTHORS | 3 +- doc/modules.rst | 7 +++- doc/modules/debug.rst | 5 +++ doc/modules/event.rst | 5 +++ doc/modules/greenio.rst | 1 - doc/modules/greenthread.rst | 5 +++ doc/modules/proc.rst | 6 --- doc/modules/processes.rst | 1 - doc/modules/queue.rst | 5 +++ doc/modules/saranwrap.rst | 1 - doc/modules/{coros.rst => semaphore.rst} | 4 +- eventlet/debug.py | 27 ++++++++++---- eventlet/event.py | 39 +++++++++++--------- eventlet/greenpool.py | 31 ++++++++-------- eventlet/greenthread.py | 47 +++++++++++++++--------- eventlet/hubs/__init__.py | 3 +- eventlet/processes.py | 1 + tests/debug_test.py | 1 + 18 files changed, 120 insertions(+), 72 deletions(-) create mode 100644 doc/modules/debug.rst create mode 100644 doc/modules/event.rst create mode 100644 doc/modules/greenthread.rst delete mode 100644 doc/modules/proc.rst create mode 100644 doc/modules/queue.rst rename doc/modules/{coros.rst => semaphore.rst} (50%) diff --git a/AUTHORS b/AUTHORS index abe7e53..227ce9f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,6 +7,7 @@ Contributors ------------ * Denis Bilenko, AG Projects * Mike Barton +* R. Tyler Ballance * Chet Murthy * radix @@ -24,13 +25,11 @@ Linden Lab Contributors Thanks To --------- * AdamKG, giving the hint that invalid argument errors were introduced post-0.9.0 -* Michael Barton, 100-continue patch, content-length bugfixes for wsgi, reviews of wsgi logic * gholt, wsgi patches for accepting a custom pool, and returning 400 if content-length is invalid * Luke Tucker, bug report regarding wsgi + webob * Chuck Thier, reporting a bug in processes.py * Brantley Harris, reporting bug #4 * Taso Du Val, reproing an exception squelching bug, saving children's lives ;-) -* R. Tyler Ballance, bug report on tpool on Windows, help with improving corolocal module, profile performance report, suggestion that fixed tpool on Windows, quick reporting of wsgi bug, sharing production experiences, patch to support keepalive-disabling * Sergey Shepelev, PEP 8 police :-), reporting bug #5 * Luci Stanescu, for reporting twisted hub bug * Marcus Cavanaugh, for test case code that has been incredibly useful in tracking down bugs diff --git a/doc/modules.rst b/doc/modules.rst index 2bdcad1..17f9e5b 100644 --- a/doc/modules.rst +++ b/doc/modules.rst @@ -7,13 +7,16 @@ Module Reference modules/api modules/backdoor modules/corolocal - modules/coros + modules/debug modules/db_pool + modules/event modules/greenio modules/greenpool + modules/greenthread modules/pools modules/processes - modules/proc + modules/queue modules/saranwrap + modules/semaphore modules/util modules/wsgi diff --git a/doc/modules/debug.rst b/doc/modules/debug.rst new file mode 100644 index 0000000..03c2ddc --- /dev/null +++ b/doc/modules/debug.rst @@ -0,0 +1,5 @@ +:mod:`debug` -- Debugging tools for Eventlet +================================================== + +.. automodule:: eventlet.debug + :members: diff --git a/doc/modules/event.rst b/doc/modules/event.rst new file mode 100644 index 0000000..6774671 --- /dev/null +++ b/doc/modules/event.rst @@ -0,0 +1,5 @@ +:mod:`event` -- Cross-greenthread primitive +================================================== + +.. automodule:: eventlet.event + :members: diff --git a/doc/modules/greenio.rst b/doc/modules/greenio.rst index 2b5909b..05ca405 100644 --- a/doc/modules/greenio.rst +++ b/doc/modules/greenio.rst @@ -3,4 +3,3 @@ .. automodule:: eventlet.greenio :members: - :undoc-members: diff --git a/doc/modules/greenthread.rst b/doc/modules/greenthread.rst new file mode 100644 index 0000000..8367402 --- /dev/null +++ b/doc/modules/greenthread.rst @@ -0,0 +1,5 @@ +:mod:`greenthread` -- Green Thread Implementation +================================================== + +.. automodule:: eventlet.greenthread + :members: diff --git a/doc/modules/proc.rst b/doc/modules/proc.rst deleted file mode 100644 index f20433d..0000000 --- a/doc/modules/proc.rst +++ /dev/null @@ -1,6 +0,0 @@ -:mod:`proc` -- Advanced coroutine control -========================================== - -.. automodule:: eventlet.proc - :members: - :undoc-members: diff --git a/doc/modules/processes.rst b/doc/modules/processes.rst index 3669080..6ae5b15 100644 --- a/doc/modules/processes.rst +++ b/doc/modules/processes.rst @@ -3,4 +3,3 @@ .. automodule:: eventlet.processes :members: - :undoc-members: diff --git a/doc/modules/queue.rst b/doc/modules/queue.rst new file mode 100644 index 0000000..9c3a933 --- /dev/null +++ b/doc/modules/queue.rst @@ -0,0 +1,5 @@ +:mod:`queue` -- Queue class +======================================== + +.. automodule:: eventlet.queue + :members: diff --git a/doc/modules/saranwrap.rst b/doc/modules/saranwrap.rst index c9f1802..0d328fd 100644 --- a/doc/modules/saranwrap.rst +++ b/doc/modules/saranwrap.rst @@ -18,7 +18,6 @@ The objects so wrapped behave as if they are resident in the current process spa .. automodule:: eventlet.saranwrap :members: - :undoc-members: Underlying Protocol diff --git a/doc/modules/coros.rst b/doc/modules/semaphore.rst similarity index 50% rename from doc/modules/coros.rst rename to doc/modules/semaphore.rst index 0778279..7d92ee4 100644 --- a/doc/modules/coros.rst +++ b/doc/modules/semaphore.rst @@ -1,6 +1,6 @@ -:mod:`coros` -- Coroutine communication patterns +:mod:`semaphore` -- Semaphore classes ================================================== -.. automodule:: eventlet.coros +.. automodule:: eventlet.semaphore :members: :undoc-members: diff --git a/eventlet/debug.py b/eventlet/debug.py index 239a137..87a1d0e 100644 --- a/eventlet/debug.py +++ b/eventlet/debug.py @@ -1,9 +1,10 @@ """The debug module contains utilities and functions for better debugging Eventlet-powered applications.""" -__all__ = ['spew', 'unspew', 'hub_listener_stacks', -'hub_exceptions', 'tpool_exceptions'] +import os +__all__ = ['spew', 'unspew', 'format_hub_listeners', 'hub_listener_stacks', +'hub_exceptions', 'tpool_exceptions'] class Spew(object): """ @@ -60,15 +61,27 @@ def unspew(): sys.settrace(None) +def format_hub_listeners(): + """ Returns a formatted string of the current listeners on the current + hub. This can be useful in determining what's going on in the event system, + especially when used in conjunction with :func:`hub_listener_stacks`. + """ + from eventlet import hubs + hub = hubs.get_hub() + result = ['READERS:'] + for l in hub.get_readers(): + result.append(repr(l)) + result.append('WRITERS:') + for l in hub.get_writers(): + result.append(repr(l)) + return os.linesep.join(result) + def hub_listener_stacks(state): """Toggles whether or not the hub records the stack when clients register listeners on file descriptors. This can be useful when trying to figure out what the hub is up to at any given moment. To inspect the stacks - of the current listeners, you have to iterate over them:: - - from eventlet import hubs - for reader in hubs.get_hub().get_readers(): - print reader + of the current listeners, call :func:`format_hub_listeners` at critical + junctures in the application logic. """ from eventlet import hubs hubs.get_hub().set_debug_listeners(state) diff --git a/eventlet/event.py b/eventlet/event.py index fe5d5cc..a251e30 100644 --- a/eventlet/event.py +++ b/eventlet/event.py @@ -13,20 +13,22 @@ class Event(object): """An abstraction where an arbitrary number of coroutines can wait for one event from another. - Events differ from channels in two ways: + Events are similar to a Queue that can only hold one item, but differ + in two important ways: - 1. calling :meth:`send` does not unschedule the current coroutine + 1. calling :meth:`send` never unschedules the current greenthread 2. :meth:`send` can only be called once; use :meth:`reset` to prepare the event for another :meth:`send` - They are ideal for communicating return values between coroutines. + They are good for communicating results between coroutines. - >>> from eventlet import coros, api - >>> evt = coros.Event() + >>> from eventlet import event + >>> import eventlet + >>> evt = event.Event() >>> def baz(b): ... evt.send(b + 1) ... - >>> _ = api.spawn(baz, 3) + >>> _ = eventlet.spawn_n(baz, 3) >>> evt.wait() 4 """ @@ -43,8 +45,8 @@ class Event(object): """ Reset this event so it can be used to send again. Can only be called after :meth:`send` has been called. - >>> from eventlet import coros - >>> evt = coros.Event() + >>> from eventlet import event + >>> evt = event.Event() >>> evt.send(1) >>> evt.reset() >>> evt.send(2) @@ -103,14 +105,15 @@ class Event(object): Returns the value the other coroutine passed to :meth:`send`. - >>> from eventlet import coros, api - >>> evt = coros.Event() + >>> from eventlet import event + >>> import eventlet + >>> evt = event.Event() >>> def wait_on(): ... retval = evt.wait() ... print "waited for", retval - >>> _ = api.spawn(wait_on) + >>> _ = eventlet.spawn(wait_on) >>> evt.send('result') - >>> api.sleep(0) + >>> eventlet.sleep(0) waited for result Returns immediately if the event has already @@ -134,17 +137,18 @@ class Event(object): """Makes arrangements for the waiters to be woken with the result and then returns immediately to the parent. - >>> from eventlet import coros, api - >>> evt = coros.Event() + >>> from eventlet import event + >>> import eventlet + >>> evt = event.Event() >>> def waiter(): ... print 'about to wait' ... result = evt.wait() ... print 'waited for', result - >>> _ = api.spawn(waiter) - >>> api.sleep(0) + >>> _ = eventlet.spawn(waiter) + >>> eventlet.sleep(0) about to wait >>> evt.send('a') - >>> api.sleep(0) + >>> eventlet.sleep(0) waited for a It is an error to call :meth:`send` multiple times on the same event. @@ -175,5 +179,6 @@ class Event(object): waiter.throw(*exc) def send_exception(self, *args): + """Same as :meth:`send`, but sends an exception to waiters.""" # the arguments and the same as for greenlet.throw return self.send(None, args) \ No newline at end of file diff --git a/eventlet/greenpool.py b/eventlet/greenpool.py index 0590d1b..1c8479b 100644 --- a/eventlet/greenpool.py +++ b/eventlet/greenpool.py @@ -27,12 +27,12 @@ class GreenPool(object): self.no_coros_running = event.Event() def resize(self, new_size): - """ Change the max number of coroutines doing work at any given time. + """ Change the max number of greenthreads doing work at any given time. - If resize is called when there are more than *new_size* - coroutines already working on tasks, they will be allowed to complete - but no new tasks will be allowed to get launched until enough coroutines - finish their tasks to drop the overall quantity below *new_size*. Until + If resize is called when there are more than *new_size* greenthreads + already working on tasks, they will be allowed to complete but no new + tasks will be allowed to get launched until enough greenthreads finish + their tasks to drop the overall quantity below *new_size*. Until then, the return value of free() will be negative. """ size_delta = new_size - self.size @@ -40,15 +40,15 @@ class GreenPool(object): self.size = new_size def running(self): - """ Returns the number of coroutines that are currently executing + """ Returns the number of greenthreads that are currently executing functions in the Parallel's pool.""" return len(self.coroutines_running) def free(self): - """ Returns the number of coroutines available for use. + """ Returns the number of greenthreads available for use. - If zero or less, the next call to :meth:`spawn` will block the calling - coroutine until a slot becomes available.""" + If zero or less, the next call to :meth:`spawn` or :meth:`spawn_n` will + block the calling greenthread until a slot becomes available.""" return self.sem.counter def spawn(self, function, *args, **kwargs): @@ -89,8 +89,8 @@ class GreenPool(object): self._spawn_done(coro) def spawn_n(self, func, *args, **kwargs): - """ Create a coroutine to run the *function*. Returns None; the results - of the function are not retrievable. + """ Create a greenthread to run the *function*. Returns None; the + results of the function are not retrievable. """ # if reentering an empty pool, don't try to wait on a coroutine freeing # itself -- instead, just execute in the current coroutine @@ -99,13 +99,14 @@ class GreenPool(object): self._spawn_n_impl(func, args, kwargs) else: self.sem.acquire() - g = greenthread.spawn_n(self._spawn_n_impl, func, args, kwargs, coro=True) + g = greenthread.spawn_n(self._spawn_n_impl, + func, args, kwargs, coro=True) if not self.coroutines_running: self.no_coros_running = event.Event() self.coroutines_running.add(g) def waitall(self): - """Waits until all coroutines in the pool are finished working.""" + """Waits until all greenthreads in the pool are finished working.""" self.no_coros_running.wait() def _spawn_done(self, coro): @@ -118,7 +119,7 @@ class GreenPool(object): self.no_coros_running.send(None) def waiting(self): - """Return the number of coroutines waiting to spawn. + """Return the number of greenthreads waiting to spawn. """ if self.sem.balance < 0: return -self.sem.balance @@ -194,7 +195,7 @@ class GreenPile(object): return self def next(self): - """Wait for the next result, suspending the current coroutine until it + """Wait for the next result, suspending the current greenthread until it is available. Raises StopIteration when there are no more results.""" if self.counter == 0 and self.used: raise StopIteration() diff --git a/eventlet/greenthread.py b/eventlet/greenthread.py index aa8ec97..f6f4358 100644 --- a/eventlet/greenthread.py +++ b/eventlet/greenthread.py @@ -30,8 +30,14 @@ def sleep(seconds=0): def spawn(func, *args, **kwargs): - """Create a green thread to run func(*args, **kwargs). Returns a - GreenThread object which you can use to get the results of the call. + """Create a greenthread to run ``func(*args, **kwargs)``. Returns a + :class:`GreenThread` object which you can use to get the results of the + call. + + Execution control returns immediately to the caller; the created greenthread + is merely scheduled to be run at the next available opportunity. + Use :func:`call_after_global` to arrange for greenthreads to be spawned + after a finite delay. """ hub = hubs.get_hub() g = GreenThread(hub.greenlet) @@ -46,9 +52,10 @@ def _main_wrapper(func, args, kwargs): def spawn_n(func, *args, **kwargs): - """Same as spawn, but returns a greenlet object from which it is not - possible to retrieve the results. This is slightly faster than spawn; it is - fastest if there are no keyword arguments.""" + """Same as :func:`spawn`, but returns a ``greenlet`` object from which it is + not possible to retrieve the results. This is slightly faster + than :func:`spawn` in all cases; it is fastest if there are no keyword + arguments.""" return _spawn_n(0, func, args, kwargs)[1] @@ -58,23 +65,21 @@ def call_after_global(seconds, func, *args, **kwargs): *seconds* may be specified as an integer, or a float if fractional seconds are desired. The *function* will be called with the given *args* and - keyword arguments *kwargs*, and will be executed within the main loop's - coroutine. - - Its return value is discarded. Any uncaught exception will be logged.""" + keyword arguments *kwargs*, and will be executed within its own greenthread. + + Its return value is discarded.""" return _spawn_n(seconds, func, args, kwargs)[0] def call_after_local(seconds, function, *args, **kwargs): """Schedule *function* to be called after *seconds* have elapsed. - The function will NOT be called if the current greenlet has exited. + The function will NOT be called if the current greenthread has exited. *seconds* may be specified as an integer, or a float if fractional seconds are desired. The *function* will be called with the given *args* and - keyword arguments *kwargs*, and will be executed within the main loop's - coroutine. + keyword arguments *kwargs*, and will be executed within its own greenthread. - Its return value is discarded. Any uncaught exception will be logged. + Its return value is discarded. """ hub = hubs.get_hub() g = greenlet.greenlet(_main_wrapper, parent=hub.greenlet) @@ -180,7 +185,7 @@ def _spawn_n(seconds, func, args, kwargs): class GreenThread(greenlet.greenlet): """The GreenThread class is a type of Greenlet which has the additional property of having a retrievable result. Do not construct GreenThread - objects directly; call :func:greenthread.spawn to get one. + objects directly; call :func:`spawn` to get one. """ def __init__(self, parent): greenlet.greenlet.__init__(self, self.main, parent) @@ -188,20 +193,28 @@ class GreenThread(greenlet.greenlet): def wait(self): """ Returns the result of the main function of this GreenThread. If the - result is a normal return value, wait() returns it. If it raised - an exception, wait() will also raise an exception.""" + result is a normal return value, :meth:`wait` returns it. If it raised + an exception, :meth:`wait` will raise the same exception (though the + stack trace will unavoidably contain some frames from within the + greenthread module).""" return self._exit_event.wait() def link(self, func, *curried_args, **curried_kwargs): """ Set up a function to be called with the results of the GreenThread. The function must have the following signature:: - def func(gt, [curried args/kwargs]): + + def func(gt, [curried args/kwargs]): When the GreenThread finishes its run, it calls *func* with itself and with the arguments supplied at link-time. If the function wants to retrieve the result of the GreenThread, it should call wait() on its first argument. + + Note that *func* is called within execution context of + the GreenThread, so it is possible to interfere with other linked + functions by doing things like switching explicitly to another + greenthread. """ self._exit_funcs = getattr(self, '_exit_funcs', []) self._exit_funcs.append((func, curried_args, curried_kwargs)) diff --git a/eventlet/hubs/__init__.py b/eventlet/hubs/__init__.py index 2121cfc..3f10113 100644 --- a/eventlet/hubs/__init__.py +++ b/eventlet/hubs/__init__.py @@ -3,7 +3,7 @@ import sys import threading _threadlocal = threading.local() -__all__ = ["use_hub"] +__all__ = ["use_hub", "get_hub", "get_default_hub"] def get_default_hub(): """Select the default hub implementation based on what multiplexing @@ -56,6 +56,7 @@ def use_hub(mod=None): if hasattr(_threadlocal, 'hub'): del _threadlocal.hub if isinstance(mod, str): + assert mod.strip(), "Need to specify a hub" mod = __import__('eventlet.hubs.' + mod, globals(), locals(), ['Hub']) if hasattr(mod, 'Hub'): _threadlocal.Hub = mod.Hub diff --git a/eventlet/processes.py b/eventlet/processes.py index d13bd18..cd9721c 100644 --- a/eventlet/processes.py +++ b/eventlet/processes.py @@ -47,6 +47,7 @@ def cooperative_wait(pobj, check_interval=0.01): class Process(object): + """Construct Process objects, then call read, and write on them.""" process_number = 0 def __init__(self, command, args, dead_callback=lambda:None): self.process_number = self.process_number + 1 diff --git a/tests/debug_test.py b/tests/debug_test.py index eace580..4afa349 100644 --- a/tests/debug_test.py +++ b/tests/debug_test.py @@ -18,6 +18,7 @@ class TestDebug(LimitedTestCase): debug.tpool_exceptions(False) debug.hub_listener_stacks(True) debug.hub_listener_stacks(False) + debug.format_hub_listeners() def test_hub_exceptions(self): debug.hub_exceptions(True)