diff --git a/doc/basic_usage.rst b/doc/basic_usage.rst index b67d309..fa75ca6 100644 --- a/doc/basic_usage.rst +++ b/doc/basic_usage.rst @@ -71,7 +71,7 @@ The file :ref:`echo server example ` contains a somewhat mo Primary API =========== -The design goal for Eventlet's API is simplicity and readability. You should be able to read its code and understand what it's doing. Fewer lines of code are preferred over excessively clever implementations. Like Python itself, there should be only one right way to do something with Eventlet! +The design goal for Eventlet's API is simplicity and readability. You should be able to read its code and understand what it's doing. Fewer lines of code are preferred over excessively clever implementations. `Like Python itself `_, there should be one, and only one obvious way to do it in Eventlet! Though Eventlet has many modules, much of the most-used stuff is accessible simply by doing ``import eventlet`` @@ -100,39 +100,3 @@ Though Eventlet has many modules, much of the most-used stuff is accessible simp Queues are a fundamental construct for communicating data between execution units. Eventlet's Queue class is used to communicate between greenthreads, and provides a bunch of useful features for doing that. See :class:`queue.Queue` for more details. These are the basic primitives of Eventlet; there are a lot more out there in the other Eventlet modules; check out the :doc:`modules`. - - -Green Libraries ----------------- - -The package ``eventlet.green`` contains libraries that have the same interfaces as common standard ones, but they are modified to behave well with green threads. This can be preferable than monkeypatching in many circumstances, because it may be necessary to interoperate with some module that needs the standard libraries unmolested, or simply because it's good engineering practice to be able to understand how a file behaves based simply on its contents. - -To use green libraries, simply import the desired module from ``eventlet.green``:: - - from eventlet.green import socket - from eventlet.green import threading - from eventlet.green import asyncore - -That's all there is to it! - - -Monkeypatching the Standard Library ----------------------------------------- - -.. automethod:: eventlet.util::wrap_socket_with_coroutine_socket - -Eventlet's socket object, whose implementation can be found in the -:mod:`eventlet.greenio` module, is designed to match the interface of the -standard library :mod:`socket` object. However, it is often useful to be able to -use existing code which uses :mod:`socket` directly without modifying it to use the eventlet apis. To do this, one must call :func:`~eventlet.util.wrap_socket_with_coroutine_socket`. It is only necessary -to do this once, at the beginning of the program, and it should be done before -any socket objects which will be used are created. - -.. automethod:: eventlet.util::wrap_select_with_coroutine_select - -Some code which is written in a multithreaded style may perform some tricks, -such as calling :mod:`select` with only one file descriptor and a timeout to -prevent the operation from being unbounded. For this specific situation there -is :func:`~eventlet.util.wrap_select_with_coroutine_select`; however it's -always a good idea when trying any new library with eventlet to perform some -tests to ensure eventlet is properly able to multiplex the operations. diff --git a/doc/examples.rst b/doc/examples.rst index 201d3be..7bf99b3 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -4,30 +4,35 @@ Examples Here are a bunch of small example programs that use Eventlet. All of these examples can be found in the ``examples`` directory of a source copy of Eventlet. .. _web_crawler_example: + Web Crawler ------------ .. literalinclude:: ../examples/webcrawler.py .. _wsgi_server_example: + WSGI Server ------------ .. literalinclude:: ../examples/wsgi.py .. _echo_server_example: + Echo Server ----------- .. literalinclude:: ../examples/echoserver.py .. _socket_connect_example: + Socket Connect -------------- .. literalinclude:: ../examples/connect.py .. _chat_server_example: + Multi-User Chat Server ----------------------- diff --git a/doc/index.rst b/doc/index.rst index 8373d9e..c0c86c0 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -8,7 +8,7 @@ Eventlet is different from other event-based frameworks out there because it doe Simple Example ------------------- -This is a simple web crawler that fetches a bunch of urls concurrently. +This is a simple web crawler that fetches a bunch of urls concurrently:: urls = ["http://www.google.com/intl/en_ALL/images/logo.gif", "https://wiki.secondlife.com/w/images/secondlife.jpg", @@ -23,7 +23,7 @@ This is a simple web crawler that fetches a bunch of urls concurrently. pool = eventlet.GreenPool() for body in pool.imap(fetch, urls): print "got body", len(body) - + Contents ========= @@ -32,6 +32,7 @@ Contents :maxdepth: 2 basic_usage + patching examples ssl threading diff --git a/doc/modules/api.rst b/doc/modules/api.rst deleted file mode 100644 index 6e329ae..0000000 --- a/doc/modules/api.rst +++ /dev/null @@ -1,6 +0,0 @@ -:mod:`api` -- General purpose functions -========================================== - -.. automodule:: eventlet.api - :members: - :undoc-members: diff --git a/doc/modules/greenio.rst b/doc/modules/greenio.rst deleted file mode 100644 index 05ca405..0000000 --- a/doc/modules/greenio.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`greenio` -- Cooperative network primitives -================================================= - -.. automodule:: eventlet.greenio - :members: diff --git a/doc/patching.rst b/doc/patching.rst new file mode 100644 index 0000000..94a98eb --- /dev/null +++ b/doc/patching.rst @@ -0,0 +1,49 @@ +Greening The World +================== + +One of the challenges of writing a library like Eventlet is that the built-in networking libraries don't natively support the sort of cooperative yielding that we need. What we must do instead is patch standard library modules in certain key places so that they do cooperatively yield. We've in the past considered doing this automatically upon importing Eventlet, but have decided against that course of action because it is un-Pythonic to change the behavior of module A simply by importing module B. + +Therefore, the application using Eventlet must explicitly green the world for itself, using one or both of the convenient methods provided. + +Import Green +-------------- + +The first way of greening an application is to import networking-related libraries from the ``eventlet.green`` package. It contains libraries that have the same interfaces as common standard ones, but they are modified to behave well with green threads. Using this method is a good engineering practice, because the true dependencies are apparent in every file:: + + from eventlet.green import socket + from eventlet.green import threading + from eventlet.green import asyncore + +This works best if every library can be imported green in this manner. If ``eventlet.green`` lacks a module (for example, non-python-standard modules), then the :mod:`eventlet.patcher` module can come to the rescue. It provides a function, :func:`eventlet.patcher.import_patched`, that greens any module on import. + +.. function:: eventlet.patcher.import_patched(module_name, *additional_modules, **kw_additional_modules) + + Imports a module in a greened manner, so that the module's use of networking libraries like socket will use Eventlet's green versions instead. The only required argument is the name of the module to be imported:: + + from eventlet import patcher + httplib2 = patcher.import_patched('httplib2') + + Under the hood, it works by temporarily swapping out the "normal" versions of the libraries in sys.modules for an eventlet.green equivalent. When the import of the to-be-patched module completes, the state of sys.modules is restored. Therefore, if the patched module contains the statement 'import socket', import_patched will have it reference eventlet.green.socket. One weakness of this approach is that it doesn't work for late binding (i.e. imports that happen during runtime). Late binding of imports is fortunately rarely done (it's slow and against `PEP-8 `_), so in most cases import_patched will work just fine. + + One other aspect of import_patched is the ability to specify exactly which modules are patched. Doing so may provide a slight performance benefit since only the needed modules are imported, whereas import_patched with no arguments imports a bunch of modules in case they're needed. The *additional_modules* and *kw_additional_modules* arguments are both sequences of name/module pairs. Either or both can be used:: + + from eventlet.green import socket + from eventlet.green import SocketServer + BaseHTTPServer = patcher.import_patched('BaseHTTPServer', + ('socket', socket), + ('SocketServer', SocketServer)) + BaseHTTPServer = patcher.import_patched('BaseHTTPServer', + socket=socket, SocketServer=SocketServer) + + +Monkeypatching the Standard Library +---------------------------------------- + +The other way of greening an application is simply to monkeypatch the standard +library. This has the disadvantage of appearing quite magical, but the advantage of avoiding the late-binding problem. + +.. function:: eventlet.patcher.monkey_patch(os=True, select=True, socket=True, thread=True, time=True) + + By default, this function monkeypatches the key system modules by replacing their key elements with green equivalents. The keyword arguments afford some control over which modules are patched, in case that's important. For almost all of them, they patch the single module of the same name (e.g. time=True means that the time module is patched [time.sleep is patched by eventlet.sleep]). The exceptions to this rule are *socket*, which also patches the :mod:`ssl` module if present; and *thread*, which patches both :mod:`thread` and :mod:`Queue`. + + It is important to call :func:`eventlet.patcher.monkey_patch` as early in the lifetime of the application as possible. Try to do it as one of the first lines in the main module. The reason for this is that sometimes there is a class that inherits from a class that needs to be greened -- e.g. a class that inherits from socket.socket -- and inheritance is done at import time, so therefore the monkeypatching should happen before the module that has the derived class is imported. \ No newline at end of file diff --git a/doc/real_index.html b/doc/real_index.html index 4729811..bbc7ad5 100644 --- a/doc/real_index.html +++ b/doc/real_index.html @@ -100,11 +100,12 @@ easy_install eventlet

Links

diff --git a/eventlet/green/Queue.py b/eventlet/green/Queue.py index b2efead..6767043 100644 --- a/eventlet/green/Queue.py +++ b/eventlet/green/Queue.py @@ -2,6 +2,8 @@ from eventlet import queue __all__ = ['Empty', 'Full', 'LifoQueue', 'PriorityQueue', 'Queue'] +__patched__ = ['LifoQueue', 'PriorityQueue', 'Queue'] + # these classes exist to paper over the major operational difference between # eventlet.queue.Queue and the stdlib equivalents class Queue(queue.Queue): diff --git a/eventlet/green/os.py b/eventlet/green/os.py index 1798d36..db5fa27 100644 --- a/eventlet/green/os.py +++ b/eventlet/green/os.py @@ -6,6 +6,8 @@ from eventlet import greenio from eventlet import greenthread from eventlet import hubs +__patched__ = ['fdopen', 'read', 'write', 'wait', 'waitpid'] + for var in dir(os_orig): exec "%s = os_orig.%s" % (var, var) diff --git a/eventlet/green/select.py b/eventlet/green/select.py index 48f6d78..29e2bdd 100644 --- a/eventlet/green/select.py +++ b/eventlet/green/select.py @@ -5,6 +5,8 @@ error = __select.error from eventlet.api import getcurrent from eventlet.hubs import get_hub +__patched__ = ['select'] + def get_fileno(obj): # The purpose of this function is to exactly replicate # the behavior of the select module when confronted with diff --git a/eventlet/green/socket.py b/eventlet/green/socket.py index 866f1eb..d5aa063 100644 --- a/eventlet/green/socket.py +++ b/eventlet/green/socket.py @@ -10,6 +10,9 @@ import os import sys import warnings +__patched__ = ['fromfd', 'socketpair', 'gethostbyname', 'create_connection', + 'ssl', 'socket'] + def fromfd(*args): return socket(__socket.fromfd(*args)) diff --git a/eventlet/green/ssl.py b/eventlet/green/ssl.py index de2dac3..9c15541 100644 --- a/eventlet/green/ssl.py +++ b/eventlet/green/ssl.py @@ -13,6 +13,7 @@ orig_socket = __import__('socket') socket = orig_socket.socket timeout_exc = orig_socket.timeout +__patched__ = ['SSLSocket', 'wrap_socket', 'sslwrap_simple'] class GreenSSLSocket(__ssl.SSLSocket): """ This is a green version of the SSLSocket class from the ssl module added diff --git a/eventlet/green/thread.py b/eventlet/green/thread.py index b5fe8fb..cf1cb31 100644 --- a/eventlet/green/thread.py +++ b/eventlet/green/thread.py @@ -4,6 +4,9 @@ from eventlet.support import greenlets as greenlet from eventlet.api import spawn from eventlet.semaphore import Semaphore as LockType +__patched__ = ['get_ident', 'start_new_thread', 'start_new', 'allocate_lock', + 'allocate', 'exit', 'interrupt_main', 'stack_size', '_local'] + error = __thread.error def get_ident(gr=None): diff --git a/eventlet/green/time.py b/eventlet/green/time.py index 95a845f..35c110c 100644 --- a/eventlet/green/time.py +++ b/eventlet/green/time.py @@ -1,4 +1,5 @@ __time = __import__('time') for var in dir(__time): exec "%s = __time.%s" % (var, var) +__patched__ = ['sleep'] from eventlet.api import sleep diff --git a/eventlet/patcher.py b/eventlet/patcher.py index d7f911e..64f9cf9 100644 --- a/eventlet/patcher.py +++ b/eventlet/patcher.py @@ -1,17 +1,33 @@ import sys +__all__ = ['inject', 'import_patched', 'monkey_patch'] + __exclude = set(('__builtins__', '__file__', '__name__')) - def inject(module_name, new_globals, *additional_modules): + """Base method for "injecting" greened modules into an imported module. It + imports the module specified in *module_name*, arranging things so + that the already-imported modules in *additional_modules* are used when + *module_name* makes its imports. + + *new_globals* is either None or a globals dictionary that gets populated + with the contents of the *module_name* module. This is useful when creating + a "green" version of some other module. + + *additional_modules* should be a collection of two-element tuples, of the + form (, ). If it's not specified, a default selection of + name/module pairs is used, which should cover all use cases but may be + slower because there are inevitably redundant or unnecessary imports. + """ if not additional_modules: # supply some defaults additional_modules = ( _green_os_modules() + _green_select_modules() + _green_socket_modules() + - _green_thread_modules()) + _green_thread_modules() + + _green_time_modules()) ## Put the specified modules in sys.modules for the duration of the import saved = {} @@ -50,6 +66,12 @@ def inject(module_name, new_globals, *additional_modules): def import_patched(module_name, *additional_modules, **kw_additional_modules): + """Imports a module in a way that ensures that the module uses "green" + versions of the standard library modules, so that everything works + nonblockingly. + + The only required argument is the name of the module to be imported. + """ return inject( module_name, None, @@ -76,7 +98,14 @@ def patch_function(func, *additional_modules): return patched -def monkey_patch(os=True, select=True, socket=True, thread=True): +def monkey_patch(os=True, select=True, socket=True, thread=True, time=True): + """Globally patches certain system modules to be greenthread-friendly. + + The keyword arguments afford some control over which modules are patched. + For almost all of them, they patch the single module of the same name. The + exceptions are socket, which also patches the ssl module if present; and + thread, which patches thread and Queue. + """ modules_to_patch = [] if os: modules_to_patch += _green_os_modules() @@ -86,13 +115,18 @@ def monkey_patch(os=True, select=True, socket=True, thread=True): modules_to_patch += _green_socket_modules() if thread: modules_to_patch += _green_thread_modules() + if time: + modules_to_patch += _green_time_modules() for name, mod in modules_to_patch: - sys.modules[name] = mod + for attr in mod.__patched__: + patched_attr = getattr(mod, attr, None) + if patched_attr is not None: + setattr(sys.modules[name], attr, patched_attr) def _green_os_modules(): from eventlet.green import os return [('os', os)] - + def _green_select_modules(): from eventlet.green import select return [('select', select)] @@ -109,4 +143,7 @@ def _green_thread_modules(): from eventlet.green import Queue from eventlet.green import thread return [('Queue', Queue), ('thread', thread)] - \ No newline at end of file + +def _green_time_modules(): + from eventlet.green import time + return [('time', time)] diff --git a/tests/patcher_test.py b/tests/patcher_test.py index 060cf49..d1695a6 100644 --- a/tests/patcher_test.py +++ b/tests/patcher_test.py @@ -92,7 +92,7 @@ from eventlet import patcher patcher.monkey_patch() import socket import urllib -print "newmod", socket, urllib.socket.socket +print "newmod", socket.socket, urllib.socket.socket """ self.write_to_tempfile("newmod", new_mod) python_path = os.pathsep.join(sys.path + [self.tempdir]) @@ -104,5 +104,4 @@ print "newmod", socket, urllib.socket.socket output = p.communicate() lines = output[0].split("\n") self.assert_(lines[0].startswith('newmod')) - self.assert_('eventlet.green.socket' in lines[0]) - self.assert_('GreenSocket' in lines[0]) + self.assertEqual(lines[0].count('GreenSocket'), 2) \ No newline at end of file