From c268aaf6d10fb45ef01a9dab6245369ca96f2292 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 21 Feb 2010 09:36:07 -0500 Subject: [PATCH 1/4] Implementations of connect and listen from Sergey Shepelev. --- eventlet/greenio.py | 79 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) mode change 100644 => 100755 eventlet/greenio.py diff --git a/eventlet/greenio.py b/eventlet/greenio.py old mode 100644 new mode 100755 index a575e8c..bf61135 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -501,4 +501,83 @@ def shutdown_safe(sock): # this will often be the case in an http server context if e[0] != errno.ENOTCONN: raise + + +def connect(addr, family=socket.AF_INET, bind=None): + """Convenience function for opening client sockets. + + :param addr: Address of the server to connect to. For TCP sockets, + this is a (host, port) tuple. + :param family: Socket family, optional. See :mod:`socket` + documentation for available families. + :param bind: Local address to bind to, optional. + :return The connected green socket object. + """ + sock = GreenSocket(family, socket.SOCK_STREAM) + if bind is not None: + sock.bind(bind) + sock.connect(addr) + return sock + + +def listen(addr, family=socket.AF_INET, backlog=50): + """Convenience function for opening server sockets. This + socket can be used as the argument to :func:`serve`, or + directly by setting up an ``accept()`` loop. + Sets SO_REUSEADDR on the socket to save on annoyance. + + :param addr: Address to listen on. For TCP sockets, this is a + (host, port) tuple. + :param family: Socket family, optional. See :mod:`socket` + documentation for available families. + :param backlog: The maximum number of queued connections. Should be + at least 1; the maximum value is system-dependent. + :return The listening green socket object. + """ + sock = GreenSocket(family, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(addr) + sock.listen(backlog) + return sock + + +def wrap_ssl(sock, keyfile=None, certfile=None, server_side=False, + cert_reqs=None, ssl_version=None, ca_certs=None, + do_handshake_on_connect=True, suppress_ragged_eofs=True): + """Convenience function for converting a regular socket into an SSL + socket. Has the same interface as :func:`ssl.wrap_socket`, but + works on 2.5 or earlier, using PyOpenSSL. + + The preferred idiom is to call wrap_ssl directly on the creation + method, e.g., ``wrap_ssl(connect(addr))`` or + ``wrap_ssl(listen(addr), server_side=True)``. This way there is + no "naked" socket sitting around to accidentally corrupt the SSL + session. + + :return Green SSL object. + """ + pass + + +def serve(sock, handle, concurrency=1000): + """Runs a server on the supplied socket. Calls the function + *handle* in a separate greenthread for every incoming request. + This function blocks the calling greenthread; it won't return until + the server completes. If you desire an immediate return, + spawn a new greenthread for :func:`serve`. + + The *handle* function must raise an EndServerException to + gracefully terminate the server -- that's the only way to get the + server() function to return. Any other uncaught exceptions raised + in *handle* are raised as exceptions from :func:`serve`, so be + sure to do a good job catching exceptions that your application + raises. The return value of *handle* is ignored. + + The value in *concurrency* controls the maximum number of + greenthreads that will be open at any time handling requests. When + the server hits the concurrency limit, it stops accepting new + connections until the existing ones complete. + """ + pass + \ No newline at end of file From 9a3f2cca1253c535e718f723b9c2aca1ea0a2ac2 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 21 Feb 2010 09:57:29 -0500 Subject: [PATCH 2/4] Putting connect and listen into eventlet; fixing up the way the docs look when rendered, removed reference to serve in listen's doc since it's not complete yet. --- doc/basic_usage.rst | 5 +++ doc/modules/processes.rst | 5 --- doc/modules/saranwrap.rst | 83 --------------------------------------- doc/modules/util.rst | 6 --- eventlet/__init__.py | 4 ++ eventlet/greenio.py | 22 ++++------- 6 files changed, 17 insertions(+), 108 deletions(-) mode change 100644 => 100755 doc/basic_usage.rst delete mode 100644 doc/modules/processes.rst delete mode 100644 doc/modules/saranwrap.rst delete mode 100644 doc/modules/util.rst mode change 100644 => 100755 eventlet/__init__.py diff --git a/doc/basic_usage.rst b/doc/basic_usage.rst old mode 100644 new mode 100755 index fb18972..c7a68e1 --- a/doc/basic_usage.rst +++ b/doc/basic_usage.rst @@ -31,6 +31,10 @@ Though Eventlet has many modules, much of the most-used stuff is accessible simp Suspends the current greenthread and allows others a chance to process. See :func:`sleep ` for more details. +.. autofunction:: eventlet.connect + +.. autofunction:: eventlet.listen + .. class:: eventlet.GreenPool Pools control concurrency. It's very common in applications to want to consume only a finite amount of memory, or to restrict the amount of connections that one part of the code holds open so as to leave more for the rest, or to behave consistently in the face of unpredictable input data. GreenPools provide this control. See :class:`GreenPool ` for more on how to use these. @@ -58,5 +62,6 @@ Though Eventlet has many modules, much of the most-used stuff is accessible simp Globally patches certain system modules to be greenthread-friendly. The keyword arguments afford some control over which modules are patched. If *all* is True, then all modules are patched regardless of the other arguments. If it's False, then the rest of the keyword arguments control patching of specific subsections of the standard library. Most patch the single module of the same name (os, time, select). The exceptions are socket, which also patches the ssl module if present; and thread, which patches thread, threading, and Queue. It's safe to call monkey_patch multiple times. For more information see :ref:`monkey-patch`. + These are the basic primitives of Eventlet; there are a lot more out there in the other Eventlet modules; check out the :doc:`modules`. diff --git a/doc/modules/processes.rst b/doc/modules/processes.rst deleted file mode 100644 index 6ae5b15..0000000 --- a/doc/modules/processes.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`processes` -- Running child processes -============================================= - -.. automodule:: eventlet.processes - :members: diff --git a/doc/modules/saranwrap.rst b/doc/modules/saranwrap.rst deleted file mode 100644 index 0d328fd..0000000 --- a/doc/modules/saranwrap.rst +++ /dev/null @@ -1,83 +0,0 @@ -:mod:`saranwrap` -- Running code in separate processes -======================================================= - -This is a convenient way of bundling code off into a separate process. If you are using Python 2.6, the multiprocessing module probably suits your needs better than saranwrap will. - -The simplest way to use saranwrap is to wrap a module and then call functions on that module:: - - >>> from eventlet import saranwrap - >>> import time - >>> s_time = saranwrap.wrap(time) - >>> timeobj = s_time.gmtime(0) - >>> timeobj - saran:(1970, 1, 1, 0, 0, 0, 3, 1, 0) - >>> timeobj.tm_sec - 0 - -The objects so wrapped behave as if they are resident in the current process space, but every attribute access and function call is passed over a nonblocking pipe to the child process. For efficiency, it's best to make as few attribute calls as possible relative to the amount of work being delegated to the child process. - -.. automodule:: eventlet.saranwrap - :members: - - -Underlying Protocol -------------------- - -Saranwrap's remote procedure calls are achieved by intercepting the basic -getattr and setattr calls in a client proxy, which commnicates those -down to the server which will dispatch them to objects in it's process -space. - -The basic protocol to get and set attributes is for the client proxy -to issue the command:: - - getattr $id $name - setattr $id $name $value - - getitem $id $item - setitem $id $item $value - eq $id $rhs - del $id - -When the get returns a callable, the client proxy will provide a -callable proxy which will invoke a remote procedure call. The command -issued from the callable proxy to server is:: - - call $id $name $args $kwargs - -If the client supplies an id of None, then the get/set/call is applied -to the object(s) exported from the server. - -The server will parse the get/set/call, take the action indicated, and -return back to the caller one of:: - - value $val - callable - object $id - exception $excp - -To handle object expiration, the proxy will instruct the rpc server to -discard objects which are no longer in use. This is handled by -catching proxy deletion and sending the command:: - - del $id - -The server will handle this by removing clearing it's own internal -references. This does not mean that the object will necessarily be -cleaned from the server, but no artificial references will remain -after successfully completing. On completion, the server will return -one of:: - - value None - exception $excp - -The server also accepts a special command for debugging purposes:: - - status - -Which will be intercepted by the server to write back:: - - status {...} - -The wire protocol is to pickle the Request class in this file. The -request class is basically an action and a map of parameters. diff --git a/doc/modules/util.rst b/doc/modules/util.rst deleted file mode 100644 index d573682..0000000 --- a/doc/modules/util.rst +++ /dev/null @@ -1,6 +0,0 @@ -:mod:`util` -- Stdlib wrapping and compatibility functions -=========================================================== - -.. automodule:: eventlet.util - :members: - :undoc-members: diff --git a/eventlet/__init__.py b/eventlet/__init__.py old mode 100644 new mode 100755 index 7810e96..57a73ac --- a/eventlet/__init__.py +++ b/eventlet/__init__.py @@ -7,6 +7,7 @@ try: from eventlet import queue from eventlet import timeout from eventlet import patcher + from eventlet import greenio sleep = greenthread.sleep spawn = greenthread.spawn @@ -24,6 +25,9 @@ try: import_patched = patcher.import_patched monkey_patch = patcher.monkey_patch + connect = greenio.connect + listen = greenio.listen + # deprecated TimeoutError = timeout.Timeout exc_after = greenthread.exc_after diff --git a/eventlet/greenio.py b/eventlet/greenio.py index bf61135..bcf03d9 100755 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -506,12 +506,10 @@ def shutdown_safe(sock): def connect(addr, family=socket.AF_INET, bind=None): """Convenience function for opening client sockets. - :param addr: Address of the server to connect to. For TCP sockets, - this is a (host, port) tuple. - :param family: Socket family, optional. See :mod:`socket` - documentation for available families. + :param addr: Address of the server to connect to. For TCP sockets, this is a (host, port) tuple. + :param family: Socket family, optional. See :mod:`socket` documentation for available families. :param bind: Local address to bind to, optional. - :return The connected green socket object. + :return: The connected green socket object. """ sock = GreenSocket(family, socket.SOCK_STREAM) if bind is not None: @@ -522,18 +520,14 @@ def connect(addr, family=socket.AF_INET, bind=None): def listen(addr, family=socket.AF_INET, backlog=50): """Convenience function for opening server sockets. This - socket can be used as the argument to :func:`serve`, or - directly by setting up an ``accept()`` loop. + socket can be used in an ``accept()`` loop. Sets SO_REUSEADDR on the socket to save on annoyance. - :param addr: Address to listen on. For TCP sockets, this is a - (host, port) tuple. - :param family: Socket family, optional. See :mod:`socket` - documentation for available families. - :param backlog: The maximum number of queued connections. Should be - at least 1; the maximum value is system-dependent. - :return The listening green socket object. + :param addr: Address to listen on. For TCP sockets, this is a (host, port) tuple. + :param family: Socket family, optional. See :mod:`socket` documentation for available families. + :param backlog: The maximum number of queued connections. Should be at least 1; the maximum value is system-dependent. + :return: The listening green socket object. """ sock = GreenSocket(family, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) From fa3a1a0a053a1c6b32741470c0668a16a9405458 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 21 Feb 2010 10:20:08 -0500 Subject: [PATCH 3/4] Used convenience functions in examples. --- examples/chat_server.py | 5 +---- examples/echoserver.py | 6 +----- examples/feedscraper.py | 7 +------ examples/websocket.py | 6 +----- examples/wsgi.py | 8 ++------ 5 files changed, 6 insertions(+), 26 deletions(-) mode change 100644 => 100755 examples/chat_server.py mode change 100644 => 100755 examples/echoserver.py mode change 100644 => 100755 examples/feedscraper.py mode change 100644 => 100755 examples/websocket.py mode change 100644 => 100755 examples/wsgi.py diff --git a/examples/chat_server.py b/examples/chat_server.py old mode 100644 new mode 100755 index 3897c0f..bf24c98 --- a/examples/chat_server.py +++ b/examples/chat_server.py @@ -1,5 +1,4 @@ import eventlet -from eventlet.green import socket participants = [] @@ -17,9 +16,7 @@ def read_chat_forever(writer, reader): try: print "ChatServer starting up on port 3000" - server = socket.socket() - server.bind(('0.0.0.0', 3000)) - server.listen(50) + server = eventlet.listen(('0.0.0.0', 3000)) while True: new_connection, address = server.accept() print "Participant joined chat." diff --git a/examples/echoserver.py b/examples/echoserver.py old mode 100644 new mode 100755 index 3c367a8..0888688 --- a/examples/echoserver.py +++ b/examples/echoserver.py @@ -11,7 +11,6 @@ and then 'quit') """ import eventlet -from eventlet.green import socket def handle(fd): print "client connected" @@ -25,10 +24,7 @@ def handle(fd): print "client disconnected" print "server socket listening on port 6000" -server = socket.socket() -server.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR, 1) -server.bind(('0.0.0.0', 6000)) -server.listen(50) +server = eventlet.listen(('0.0.0.0', 6000)) pool = eventlet.GreenPool() while True: try: diff --git a/examples/feedscraper.py b/examples/feedscraper.py old mode 100644 new mode 100755 index c037145..b9e7e11 --- a/examples/feedscraper.py +++ b/examples/feedscraper.py @@ -32,9 +32,4 @@ def app(environ, start_response): if __name__ == '__main__': from eventlet import wsgi - from eventlet.green import socket - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR, 1) - sock.bind(('localhost', 9010)) - sock.listen(50) - wsgi.server(sock, app) \ No newline at end of file + wsgi.server(eventlet.listen(('localhost', 9010)), app) \ No newline at end of file diff --git a/examples/websocket.py b/examples/websocket.py old mode 100644 new mode 100755 index afeb103..2cef80c --- a/examples/websocket.py +++ b/examples/websocket.py @@ -137,9 +137,5 @@ def dispatch(environ, start_response): if __name__ == "__main__": # run an example app from the command line - from eventlet.green import socket - listener = socket.socket() - listener.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR, 1) - listener.bind(('localhost', 7000)) - listener.listen(500) + listener = eventlet.listen(('localhost', 7000)) wsgi.server(listener, dispatch) \ No newline at end of file diff --git a/examples/wsgi.py b/examples/wsgi.py old mode 100644 new mode 100755 index 93469fd..ba3f433 --- a/examples/wsgi.py +++ b/examples/wsgi.py @@ -5,8 +5,8 @@ multiple threads, and graceful code reloading, see: http://pypi.python.org/pypi/Spawning/ """ +import eventlet from eventlet import wsgi -from eventlet.green import socket def hello_world(env, start_response): if env['PATH_INFO'] != '/': @@ -15,8 +15,4 @@ def hello_world(env, start_response): start_response('200 OK', [('Content-Type', 'text/plain')]) return ['Hello, World!\r\n'] -sock = socket.socket() -sock.bind(('', 8090)) -sock.listen(500) - -wsgi.server(sock, hello_world) +wsgi.server(eventlet.listen(('', 8090)), hello_world) From bba5fcb0deec881cc7a8556032878e76d1271b6a Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 21 Feb 2010 10:58:08 -0500 Subject: [PATCH 4/4] Converted docs to use convenience functions. --- doc/design_patterns.rst | 7 ++----- doc/modules/wsgi.rst | 8 ++------ 2 files changed, 4 insertions(+), 11 deletions(-) mode change 100644 => 100755 doc/design_patterns.rst mode change 100644 => 100755 doc/modules/wsgi.rst diff --git a/doc/design_patterns.rst b/doc/design_patterns.rst old mode 100644 new mode 100755 index bcac55b..fdd8f5f --- a/doc/design_patterns.rst +++ b/doc/design_patterns.rst @@ -42,7 +42,6 @@ Server Pattern Here's a simple server-side example, a simple echo server:: import eventlet - from eventlet.green import socket def handle(client): while True: @@ -50,9 +49,7 @@ Here's a simple server-side example, a simple echo server:: if not c: break client.sendall(c) - server = socket.socket() - server.bind(('0.0.0.0', 6000)) - server.listen(50) + server = eventlet.listen(('0.0.0.0', 6000)) pool = eventlet.GreenPool(10000) while True: new_sock, address = server.accept() @@ -60,7 +57,7 @@ Here's a simple server-side example, a simple echo server:: The file :ref:`echo server example ` contains a somewhat more robust and complex version of this example. -``from eventlet.green import socket`` imports eventlet's socket module, which is just like the regular socket module, but cooperatively yielding. +``server = eventlet.listen(('0.0.0.0', 6000))`` uses a convenience function to create a listening socket. ``pool = eventlet.GreenPool(10000)`` creates a pool of green threads that could handle ten thousand clients. diff --git a/doc/modules/wsgi.rst b/doc/modules/wsgi.rst old mode 100644 new mode 100755 index 9c69db4..c05485d --- a/doc/modules/wsgi.rst +++ b/doc/modules/wsgi.rst @@ -9,17 +9,13 @@ server package. One such package is `Spawning