From ce4435bdf34a825ce7ff930955d043cb2a58ec82 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 27 May 2009 00:43:51 -0700 Subject: [PATCH 01/23] Agh, sorry, we are using both httpdate and greenlib. --- eventlet/greenlib.py | 39 +++++++++++++++++++++++++++++++++++++++ eventlet/httpdate.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 eventlet/greenlib.py create mode 100644 eventlet/httpdate.py diff --git a/eventlet/greenlib.py b/eventlet/greenlib.py new file mode 100644 index 0000000..5f21783 --- /dev/null +++ b/eventlet/greenlib.py @@ -0,0 +1,39 @@ +"""\ +@file greenlib.py +@author Bob Ippolito + +Copyright (c) 2005-2006, Bob Ippolito +Copyright (c) 2007, Linden Research, Inc. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +def switch(other=None, value=None, exc=None): + """ + Switch to another greenlet, passing value or exception + """ + self = greenlet.getcurrent() + if other is None: + other = self.parent + if other is None: + other = self + if not (other or hasattr(other, 'run')): + raise SwitchingToDeadGreenlet("Switching to dead greenlet %r %r %r" % (other, value, exc)) + if exc: + return other.throw(exc) + else: + return other.switch(value) diff --git a/eventlet/httpdate.py b/eventlet/httpdate.py new file mode 100644 index 0000000..e1f81f3 --- /dev/null +++ b/eventlet/httpdate.py @@ -0,0 +1,39 @@ +"""\ +@file httpdate.py +@author Donovan Preston + +Copyright (c) 2006-2007, Linden Research, Inc. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import time + +__all__ = ['format_date_time'] + +# Weekday and month names for HTTP date/time formatting; always English! +_weekdayname = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] +_monthname = [None, # Dummy so we can use 1-based month numbers + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] + +def format_date_time(timestamp): + year, month, day, hh, mm, ss, wd, y, z = time.gmtime(timestamp) + return "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( + _weekdayname[wd], day, _monthname[month], year, hh, mm, ss + ) From f6681936df5ce22f316054f7a64aac1c308c4588 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 27 May 2009 01:11:30 -0700 Subject: [PATCH 02/23] Port of http://svn.secondlife.com/trac/eventlet/changeset/153 --- eventlet/httpc.py | 3 ++ greentest/httpc_test.py | 64 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/eventlet/httpc.py b/eventlet/httpc.py index b713a45..b868d9f 100644 --- a/eventlet/httpc.py +++ b/eventlet/httpc.py @@ -498,6 +498,9 @@ def make_connection(scheme, location, use_proxy): # run a little heuristic to see if location is an url, and if so parse out the hostpart if location.startswith('http'): _scheme, location, path, parameters, query, fragment = url_parser(location) + + if use_proxy and scheme == 'https': + scheme = 'http' result = scheme_to_factory_map[scheme](location) result.connect() diff --git a/greentest/httpc_test.py b/greentest/httpc_test.py index dbec99f..95110e4 100644 --- a/greentest/httpc_test.py +++ b/greentest/httpc_test.py @@ -400,5 +400,69 @@ class TestHttpTime(tests.TestCase): self.assertEqual(ticks, self.secs_since_epoch) +class TestProxy(tests.TestCase): + def test_ssl_proxy(self): + def ssl_proxy(sock): + conn, addr = sock.accept() + fd = conn.makefile() + try: + line = request = fd.readline() + self.assertEqual(request, 'GET https://localhost:1234 HTTP/1.1\r\n') + while line.strip(): # eat request headers + line = fd.readline() + + # we're not going to actually proxy to localhost:1234, + # we're just going to return a response on its behalf + fd.write("HTTP/1.0 200 OK\r\n\r\n") + finally: + fd.close() + conn.close() + + server = api.tcp_listener(('0.0.0.0', 5505)) + api.spawn(ssl_proxy, server) + import os + os.environ['ALL_PROXY'] = 'localhost:5505' + httpc.get('https://localhost:1234', ok=[200], use_proxy=True) + + def test_ssl_proxy_redirects(self): + # make sure that if the proxy returns a redirect, that httpc + # successfully follows it (this was broken at one point) + def ssl_proxy(sock): + conn, addr = sock.accept() + fd = conn.makefile() + try: + line = request = fd.readline() + self.assertEqual(request, 'GET https://localhost:1234 HTTP/1.1\r\n') + while line.strip(): # eat request headers + line = fd.readline() + + # we're not going to actually proxy to localhost:1234, + # we're just going to return a response on its behalf + fd.write("HTTP/1.0 302 Found\r\nLocation: https://localhost:1234/2\r\n\r\n") + finally: + fd.close() + conn.close() + + # second request, for /2 target + conn, addr = sock.accept() + fd = conn.makefile() + try: + line = request = fd.readline() + self.assertEqual(request, 'GET https://localhost:1234/2 HTTP/1.1\r\n') + while line.strip(): # eat request headers + line = fd.readline() + fd.write("HTTP/1.0 200 OK\r\n\r\n") + finally: + fd.close() + conn.close() + sock.close() + + server = api.tcp_listener(('0.0.0.0', 5505)) + api.spawn(ssl_proxy, server) + import os + os.environ['ALL_PROXY'] = 'localhost:5505' + httpc.get('https://localhost:1234', use_proxy=True, max_retries=1) + + if __name__ == '__main__': tests.main() From c2c5512f9e99a1e27112192fa1b037f780485ef2 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 27 May 2009 01:20:32 -0700 Subject: [PATCH 03/23] Port of http://svn.secondlife.com/trac/eventlet/changeset/157 --- eventlet/saranwrap.py | 37 +++++++++++++++++++++++++++++++++---- greentest/saranwrap_test.py | 7 +++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/eventlet/saranwrap.py b/eventlet/saranwrap.py index 07e3124..8af56f3 100644 --- a/eventlet/saranwrap.py +++ b/eventlet/saranwrap.py @@ -86,8 +86,8 @@ The wire protocol is to pickle the Request class in this file. The request class is basically an action and a map of parameters' """ -import os from cPickle import dumps, loads +import os import struct import sys @@ -182,6 +182,8 @@ class Request(object): def _read_lp_hunk(stream): len_bytes = stream.read(4) + if len_bytes == '': + raise EOFError("No more data to read from %s" % stream) length = struct.unpack('I', len_bytes)[0] body = stream.read(length) return body @@ -247,6 +249,7 @@ def _unmunge_attr_name(name): name = name[len('_Proxy'):] if(name.startswith('_ObjectProxy')): name = name[len('_ObjectProxy'):] + return name class ChildProcess(object): @@ -282,6 +285,9 @@ class ChildProcess(object): return retval + def __del__(self): + self._in.close() + class Proxy(object): """\ @@ -320,7 +326,10 @@ not supported, so you have to know what has been exported. request = Request('del', {'id':dead_object}) my_cp.make_request(request) - _dead_list.remove(dead_object) + try: + _dead_list.remove(dead_object) + except KeyError: + pass # Pass all public attributes across to find out if it is # callable or a simple attribute. @@ -423,6 +432,11 @@ not need to deal with this class directly.""" def proxied_type(self): + """ Returns the type of the object in the child process. + + Calling type(obj) on a saranwrapped object will always return + , so this is a way to get at the + 'real' type value.""" if type(self) is not ObjectProxy: return type(self) @@ -431,6 +445,14 @@ def proxied_type(self): request = Request('type', {'id':my_id}) return my_cp.make_request(request) + +def getpid(self): + """ Returns the pid of the child process. The argument should be + a saranwrapped object.""" + my_cp = self.__local_dict['_cp'] + return my_cp._in.getpid() + + class CallableProxy(object): """\ @class CallableProxy @@ -527,7 +549,11 @@ when the id is None.""" _log("del %s from %s" % (id, self._objects)) # *TODO what does __del__ actually return? - del self._objects[id] + try: + del self._objects[id] + except KeyError: + pass + return None def handle_type(self, obj, req): @@ -547,7 +573,10 @@ when the id is None.""" try: str_ = _read_lp_hunk(self._in) except EOFError: - sys.exit(0) # normal exit + if _g_debug_mode: + _log("Exiting normally") + sys.exit(0) + request = loads(str_) _log("request: %s (%s)" % (request, self._objects)) req = request diff --git a/greentest/saranwrap_test.py b/greentest/saranwrap_test.py index 7ac229b..0109c05 100644 --- a/greentest/saranwrap_test.py +++ b/greentest/saranwrap_test.py @@ -338,6 +338,13 @@ sys_path = sys.path""") 'random' in obj_proxy.get_dict(), 'Coroutine in saranwrapped object did not run') + def test_child_process_death(self): + prox = saranwrap.wrap({}) + pid = saranwrap.getpid(prox) + self.assertEqual(os.kill(pid, 0), None) # assert that the process is running + del prox # removing all references to the proxy should kill the child process + self.assertRaises(OSError, os.kill, pid, 0) # raises OSError if pid doesn't exist + def test_detection_of_server_crash(self): # make the server crash here pass From e17a3feeb90fe17c04996dd5de05be6cf684dbad Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 27 May 2009 02:37:15 -0700 Subject: [PATCH 04/23] Reenabled the queue tests. The semaphore now releases to waiters in the order that they started waiting. There's a FIX note in there because the queue should not need a sleep before the waiter count is correct. --- eventlet/coros.py | 17 +++++++++++++---- greentest/test__coros_queue.py | 5 +++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/eventlet/coros.py b/eventlet/coros.py index 66e0820..a41d449 100644 --- a/eventlet/coros.py +++ b/eventlet/coros.py @@ -262,6 +262,7 @@ class Semaphore(object): def __init__(self, count=0): self.counter = count + self._order = collections.deque() self._waiters = {} def __str__(self): @@ -280,6 +281,7 @@ class Semaphore(object): return False while self.counter<=0: self._waiters[api.getcurrent()] = None + self._order.append(api.getcurrent()) try: api.get_hub().switch() finally: @@ -293,14 +295,21 @@ class Semaphore(object): def release(self, blocking=True): # `blocking' parameter is for consistency with BoundedSemaphore and is ignored self.counter += 1 - if self._waiters: + if self._waiters or self._order: api.get_hub().schedule_call_global(0, self._do_acquire) return True def _do_acquire(self): - if self._waiters and self.counter>0: - waiter, _unused = self._waiters.popitem() - waiter.switch() + if (self._waiters or self._order) and self.counter>0: + waiter = None + while not waiter and self._order: + waiter = self._order.popleft() + if waiter not in self._waiters: + waiter = None + + if waiter is not None: + self._waiters.pop(waiter) + waiter.switch() def __exit__(self, typ, val, tb): self.release() diff --git a/greentest/test__coros_queue.py b/greentest/test__coros_queue.py index 7a1d616..77939ac 100644 --- a/greentest/test__coros_queue.py +++ b/greentest/test__coros_queue.py @@ -73,7 +73,7 @@ class TestQueue(LimitedTestCase): self.assertEquals(e2.wait(),'hi') self.assertEquals(e1.wait(),'done') - def skip_test_multiple_waiters(self): + def test_multiple_waiters(self): q = coros.queue() def waiter(q, evt): @@ -164,7 +164,7 @@ class TestQueue(LimitedTestCase): self.assertEquals(e2.wait(), 'timed out') self.assertEquals(q.wait(), 'sent') - def disable_test_waiting(self): + def test_waiting(self): def do_wait(q, evt): result = q.wait() evt.send(result) @@ -175,6 +175,7 @@ class TestQueue(LimitedTestCase): api.sleep(0) self.assertEquals(1, waiting(q)) q.send('hi') + api.sleep(0) # *FIX this should not be necessary self.assertEquals(0, waiting(q)) self.assertEquals('hi', e1.wait()) self.assertEquals(0, waiting(q)) From 6f0cb1573b5abc167529770dd6279e343ca7f511 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 28 May 2009 16:25:52 -0700 Subject: [PATCH 05/23] Committing broken ssl wsgi test, in hopes that it can be fixed. --- greentest/wsgi_test.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/greentest/wsgi_test.py b/greentest/wsgi_test.py index 0ea836f..95c75ec 100644 --- a/greentest/wsgi_test.py +++ b/greentest/wsgi_test.py @@ -23,6 +23,7 @@ THE SOFTWARE. """ import cgi +import os from eventlet import api from eventlet import wsgi @@ -264,5 +265,25 @@ class TestHttpd(tests.TestCase): chunklen = int(fd.readline(), 16) self.assert_(chunks > 1) + def test_012_ssl_server(self): + from eventlet import httpc + def wsgi_app(self, environ, start_response): + print "wsgi_app" + print environ['wsgi.input'] + return [environ['wsgi.input'].read()] + + certificate_file = os.path.join(os.path.dirname(__file__), 'test_server.crt') + private_key_file = os.path.join(os.path.dirname(__file__), 'test_server.key') + + sock = api.ssl_listener(('', 4201), certificate_file, private_key_file) + + api.spawn(wsgi.server, sock, wsgi_app) + + print "pre request" + result = httpc.post("https://localhost:4201/foo", "abc") + self.assertEquals(result, 'abc') + print "post request" + + if __name__ == '__main__': tests.main() From abda8d075b82e400059aa8e79b2f14934b113e27 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 28 May 2009 19:59:52 -0700 Subject: [PATCH 06/23] Fixed failing test by adding new makefile() method to GreenSSL. --- eventlet/greenio.py | 36 +++++++++++++++++++++++++++++++++++- eventlet/httpc.py | 3 +++ greentest/wsgi_test.py | 7 ++----- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/eventlet/greenio.py b/eventlet/greenio.py index 794f64b..1dccd86 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -531,10 +531,30 @@ class GreenPipe(GreenFile): self.fd.fd.flush() + +class RefCount(object): + """ Reference counting class only to be used with GreenSSL objects """ + def __init__(self): + self._count = 1 + + def increment(self): + self._count += 1 + + def decrement(self): + self._count -= 1 + assert self._count >= 0 + + def is_referenced(self): + return self._count > 0 + + class GreenSSL(GreenSocket): - def __init__(self, fd): + def __init__(self, fd, refcount = None): GreenSocket.__init__(self, fd) self.sock = self + self._refcount = refcount + if refcount is None: + self._refcount = RefCount() read = read @@ -550,7 +570,21 @@ class GreenSSL(GreenSocket): def issuer(self): return self.fd.issuer() + def dup(self): + raise NotImplemented("Dup not supported on SSL sockets") + def makefile(self, *args, **kw): + self._refcount.increment() + return GreenFile(type(self)(self.fd, refcount = self._refcount)) + + def close(self): + self._refcount.decrement() + if self._refcount.is_referenced(): + return + super(GreenSSL, self).close() + + + def socketpair(*args): one, two = socket.socketpair(*args) diff --git a/eventlet/httpc.py b/eventlet/httpc.py index b868d9f..9e8f9c3 100644 --- a/eventlet/httpc.py +++ b/eventlet/httpc.py @@ -586,6 +586,9 @@ class HttpSuite(object): def _get_response_body(self, params, connection): if connection is None: connection = connect(params.url, params.use_proxy) + # if we're creating a new connection we know the caller + # isn't going to reuse it + params.headers['connection'] = 'close' connection.request(params.method, params.path, params.body, params.headers) params.response = connection.getresponse() diff --git a/greentest/wsgi_test.py b/greentest/wsgi_test.py index 95c75ec..a1e8a95 100644 --- a/greentest/wsgi_test.py +++ b/greentest/wsgi_test.py @@ -267,9 +267,8 @@ class TestHttpd(tests.TestCase): def test_012_ssl_server(self): from eventlet import httpc - def wsgi_app(self, environ, start_response): - print "wsgi_app" - print environ['wsgi.input'] + def wsgi_app(environ, start_response): + start_response('200 OK', {}) return [environ['wsgi.input'].read()] certificate_file = os.path.join(os.path.dirname(__file__), 'test_server.crt') @@ -279,10 +278,8 @@ class TestHttpd(tests.TestCase): api.spawn(wsgi.server, sock, wsgi_app) - print "pre request" result = httpc.post("https://localhost:4201/foo", "abc") self.assertEquals(result, 'abc') - print "post request" if __name__ == '__main__': From e297081648879db27aeb2b52df6c04ffb450b117 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 2 Jun 2009 15:06:12 -0700 Subject: [PATCH 07/23] Test and fix for weird 'Unexpected EOF' error discovered through use. wsgi_test.py may not be the best place for this test but at least it's there. --- eventlet/greenio.py | 7 +++++++ greentest/wsgi_test.py | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/eventlet/greenio.py b/eventlet/greenio.py index 1dccd86..64508d3 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -557,6 +557,13 @@ class GreenSSL(GreenSocket): self._refcount = RefCount() read = read + + def sendall(self, data): + # overriding sendall because ssl sockets behave badly when asked to + # send empty strings; 'normal' sockets don't have a problem + if not data: + return + super(GreenSSL, self).sendall(data) def write(self, data): try: diff --git a/greentest/wsgi_test.py b/greentest/wsgi_test.py index a1e8a95..f896c93 100644 --- a/greentest/wsgi_test.py +++ b/greentest/wsgi_test.py @@ -1,5 +1,5 @@ """\ -@file httpd_test.py +@file wsgi_test.py @author Donovan Preston Copyright (c) 2007, Linden Research, Inc. @@ -280,6 +280,20 @@ class TestHttpd(tests.TestCase): result = httpc.post("https://localhost:4201/foo", "abc") self.assertEquals(result, 'abc') + + def test_013_empty_return(self): + from eventlet import httpc + def wsgi_app(environ, start_response): + start_response("200 OK", []) + return [""] + + certificate_file = os.path.join(os.path.dirname(__file__), 'test_server.crt') + private_key_file = os.path.join(os.path.dirname(__file__), 'test_server.key') + sock = api.ssl_listener(('', 4202), certificate_file, private_key_file) + api.spawn(wsgi.server, sock, wsgi_app) + + res = httpc.get("https://localhost:4202/foo") + self.assertEquals(res, '') if __name__ == '__main__': From 821e88debd82bd48b79624b5e9770efa250745ed Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 2 Jun 2009 17:11:48 -0700 Subject: [PATCH 08/23] Fixed test_connect_ssl --- eventlet/greenio.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eventlet/greenio.py b/eventlet/greenio.py index 64508d3..a9531b0 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -578,11 +578,13 @@ class GreenSSL(GreenSocket): return self.fd.issuer() def dup(self): - raise NotImplemented("Dup not supported on SSL sockets") + raise NotImplementedError("Dup not supported on SSL sockets") def makefile(self, *args, **kw): self._refcount.increment() return GreenFile(type(self)(self.fd, refcount = self._refcount)) + + makeGreenFile = makefile def close(self): self._refcount.decrement() From d396d974140201498bc19b6ab1667a19cef20c59 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 2 Jun 2009 17:50:24 -0700 Subject: [PATCH 09/23] Fixed timer canceling in CoroutinePool test, this mean that each coroutine in the pool is created anew. --- eventlet/pools.py | 11 +++-------- greentest/pools_test.py | 12 ++++++------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/eventlet/pools.py b/eventlet/pools.py index 0ddbd1d..18ee6f3 100644 --- a/eventlet/pools.py +++ b/eventlet/pools.py @@ -274,7 +274,9 @@ class CoroutinePool(Pool): def _main_loop(self, sender): """ Private, infinite loop run by a pooled coroutine. """ try: - while True: + # not really a loop anymore because we want the greenlet + # to die so all its timers get canceled + if True: recvd = sender.wait() # Delete the sender's result here because the very # first event through the loop is referenced by @@ -285,21 +287,14 @@ class CoroutinePool(Pool): # tests. sender._result = coros.NOT_USED - sender = coros.event() (evt, func, args, kw) = recvd self._safe_apply(evt, func, args, kw) - #api.get_hub().cancel_timers(api.getcurrent()) # Likewise, delete these variables or else they will # be referenced by this frame until replaced by the # next recvd, which may or may not be a long time from # now. del evt, func, args, kw, recvd - - self.put(sender) finally: - # if we get here, something broke badly, and all we can really - # do is try to keep the pool from leaking items. - # Shouldn't even try to print the exception. self.put(self.create()) def _safe_apply(self, evt, func, args, kw): diff --git a/greentest/pools_test.py b/greentest/pools_test.py index 3b70560..d81c8d1 100644 --- a/greentest/pools_test.py +++ b/greentest/pools_test.py @@ -284,16 +284,16 @@ class TestCoroutinePool(tests.TestCase): self.assertEquals(['cons1', 'prod', 'cons2'], results) def test_timer_cancel(self): + timer_fired = [] + def fire_timer(): + timer_fired.append(True) def some_work(): - t = timer.Timer(5, lambda: None) - t.autocancellable = True - t.schedule() - return t + api.get_hub().schedule_call_local(0, fire_timer) pool = pools.CoroutinePool(0, 2) worker = pool.execute(some_work) - t = worker.wait() + worker.wait() api.sleep(0) - self.assertEquals(t.cancelled, True) + self.assertEquals(timer_fired, []) def test_reentrant(self): pool = pools.CoroutinePool(0,1) From d8006b3a0523628236b76e7c1740e52bb0ab2d4d Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 2 Jun 2009 17:55:18 -0700 Subject: [PATCH 10/23] Fixed processes_test breakage, and removed mode='static' from that file. --- greentest/processes_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/greentest/processes_test.py b/greentest/processes_test.py index 98ce505..b5eee6a 100644 --- a/greentest/processes_test.py +++ b/greentest/processes_test.py @@ -21,12 +21,13 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import sys + from greentest import tests from eventlet import api from eventlet import processes class TestEchoPool(tests.TestCase): - mode = 'static' def setUp(self): self.pool = processes.ProcessPool('echo', ["hello"]) @@ -50,7 +51,6 @@ class TestEchoPool(tests.TestCase): class TestCatPool(tests.TestCase): - mode = 'static' def setUp(self): self.pool = processes.ProcessPool('cat') @@ -92,7 +92,6 @@ class TestCatPool(tests.TestCase): class TestDyingProcessesLeavePool(tests.TestCase): - mode = 'static' def setUp(self): self.pool = processes.ProcessPool('echo', ['hello'], max_size=1) @@ -112,11 +111,12 @@ class TestDyingProcessesLeavePool(tests.TestCase): class TestProcessLivesForever(tests.TestCase): - mode = 'static' def setUp(self): - self.pool = processes.ProcessPool('python', ['-c', 'print "y"; import time; time.sleep(0.1); print "y"'], max_size=1) + self.pool = processes.ProcessPool(sys.executable, ['-c', 'print "y"; import time; time.sleep(0.4); print "y"'], max_size=1) def test_reading_twice_from_same_process(self): + # this test is a little timing-sensitive in that if the sub-process + # completes its sleep before we do a full put/get then it will fail proc = self.pool.get() try: result = proc.read(2) @@ -125,7 +125,7 @@ class TestProcessLivesForever(tests.TestCase): self.pool.put(proc) proc2 = self.pool.get() - self.assert_(proc is proc2) + self.assert_(proc is proc2, "This will fail if there is a timing issue") try: result = proc2.read(2) self.assertEquals(result, 'y\n') From 06482faad69eec785451646621b9b3afaf4e431f Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 2 Jun 2009 18:23:49 -0700 Subject: [PATCH 11/23] Fixed test_saranwrap --- greentest/saranwrap_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/greentest/saranwrap_test.py b/greentest/saranwrap_test.py index 0109c05..d6ad3c8 100644 --- a/greentest/saranwrap_test.py +++ b/greentest/saranwrap_test.py @@ -343,6 +343,7 @@ sys_path = sys.path""") pid = saranwrap.getpid(prox) self.assertEqual(os.kill(pid, 0), None) # assert that the process is running del prox # removing all references to the proxy should kill the child process + api.sleep(0.1) # need to let the signal handler run self.assertRaises(OSError, os.kill, pid, 0) # raises OSError if pid doesn't exist def test_detection_of_server_crash(self): From 6a27fca3ca22c4effa8ffc14e3575e1317bf5d5a Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Fri, 5 Jun 2009 14:01:31 -0700 Subject: [PATCH 12/23] Minor fix to socket method so that it returns a real socket, in a kind of abstraction-violating way. --- eventlet/httpd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eventlet/httpd.py b/eventlet/httpd.py index cc5ff9d..0111c4c 100644 --- a/eventlet/httpd.py +++ b/eventlet/httpd.py @@ -311,7 +311,7 @@ class Request(object): return self._outgoing_headers.has_key(key.lower()) def socket(self): - return self.protocol.socket + return self.protocol.rfile._sock def error(self, response=None, body=None, log_traceback=True): if log_traceback: From 25b953bb38a2d5977dee65359dd4c0d197def972 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 7 Jun 2009 10:09:39 -0700 Subject: [PATCH 13/23] Slightly better error message when malformed urls make it into make_connection --- eventlet/httpc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eventlet/httpc.py b/eventlet/httpc.py index 9e8f9c3..125a29c 100644 --- a/eventlet/httpc.py +++ b/eventlet/httpc.py @@ -510,8 +510,10 @@ def make_connection(scheme, location, use_proxy): def connect(url, use_proxy=False): """ Create a connection object to the host specified in a url. Convenience function for make_connection.""" scheme, location = url_parser(url)[:2] - return make_connection(scheme, location, use_proxy) - + try: + return make_connection(scheme, location, use_proxy) + except KeyError: + raise ValueError("Unknown url scheme %s in url %s" % (scheme, url)) def make_safe_loader(loader): if not callable(loader): From 27932cdffcc90cace784095c742619d76855b06c Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 7 Jun 2009 22:10:27 -0700 Subject: [PATCH 14/23] Nat's patch for making greenio successfully import under Windows by deferring the import of fcntl. --- eventlet/greenio.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/eventlet/greenio.py b/eventlet/greenio.py index a9531b0..6baa94f 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -32,7 +32,6 @@ import errno import os import socket from socket import socket as _original_socket -import fcntl import time @@ -169,14 +168,17 @@ def file_send(fd, data): def set_nonblocking(fd): - ## Socket - if hasattr(fd, 'setblocking'): - fd.setblocking(0) - ## File - else: + try: + setblocking = fd.setblocking + except AttributeError: + # This version of Python predates socket.setblocking() + import fcntl fileno = fd.fileno() flags = fcntl.fcntl(fileno, fcntl.F_GETFL) fcntl.fcntl(fileno, fcntl.F_SETFL, flags | os.O_NONBLOCK) + else: + # socket supports setblocking() + setblocking(0) class GreenSocket(object): From 4d7cb26915d1ee41e9a738b1abffd83b25a1ad9f Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 13 Jun 2009 13:28:24 -0700 Subject: [PATCH 15/23] Weird encoding bug in generate_report. --- greentest/generate_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/greentest/generate_report.py b/greentest/generate_report.py index f2449ef..5014d7d 100755 --- a/greentest/generate_report.py +++ b/greentest/generate_report.py @@ -205,7 +205,7 @@ def generate_raw_results(path, database): c = sqlite3.connect(database) res = c.execute('select id, stdout from command_record').fetchall() for id, out in res: - file(os.path.join(path, '%s.txt' % id), 'w').write(out) + file(os.path.join(path, '%s.txt' % id), 'w').write(out.encode('utf-8')) sys.stderr.write('.') sys.stderr.write('\n') From 740c39503c4237681b5ed2e0d0fc31dfedb7bf35 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 13 Jun 2009 13:38:23 -0700 Subject: [PATCH 16/23] Copyright header merge. --- eventlet/greenlib.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/eventlet/greenlib.py b/eventlet/greenlib.py index c1babb6..8de50e0 100644 --- a/eventlet/greenlib.py +++ b/eventlet/greenlib.py @@ -1,24 +1,23 @@ -"""\ -Copyright (c) 2005-2006, Bob Ippolito -Copyright (c) 2007, Linden Research, Inc. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +# Copyright (c) 2005-2006, Bob Ippolito +# Copyright (c) 2007, Linden Research, Inc. +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -""" from eventlet.api import Greenlet class SwitchingToDeadGreenlet(Exception): From 728f79aab583ca43b285ba97c69e12ea287b4703 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 13 Jun 2009 13:39:15 -0700 Subject: [PATCH 17/23] Re-added relevant tests to test__pool.py. They both fail, and shouldn't --- greentest/test__pool.py | 78 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/greentest/test__pool.py b/greentest/test__pool.py index 38354c7..2b5e093 100644 --- a/greentest/test__pool.py +++ b/greentest/test__pool.py @@ -39,6 +39,20 @@ class TestCoroutinePool(LimitedTestCase): done.wait() self.assertEquals(['cons1', 'prod', 'cons2'], results) + def test_timer_cancel(self): + # this test verifies that local timers are not fired + # outside of the context of the execute method + timer_fired = [] + def fire_timer(): + timer_fired.append(True) + def some_work(): + api.get_hub().schedule_call_local(0, fire_timer) + pool = self.klass(0, 2) + worker = pool.execute(some_work) + worker.wait() + api.sleep(0) + self.assertEquals(timer_fired, []) + def test_reentrant(self): pool = self.klass(0,1) def reenter(): @@ -55,6 +69,34 @@ class TestCoroutinePool(LimitedTestCase): pool.execute_async(reenter_async) evt.wait() + + def test_stderr_raising(self): + # testing that really egregious errors in the error handling code + # (that prints tracebacks to stderr) don't cause the pool to lose + # any members + import sys + pool = self.klass(min_size=1, max_size=1) + def crash(*args, **kw): + raise RuntimeError("Whoa") + class FakeFile(object): + write = crash + + # we're going to do this by causing the traceback.print_exc in + # safe_apply to raise an exception and thus exit _main_loop + normal_err = sys.stderr + try: + sys.stderr = FakeFile() + waiter = pool.execute(crash) + self.assertRaises(RuntimeError, waiter.wait) + # the pool should have something free at this point since the + # waiter returned + self.assertEqual(pool.free(), 1) + # shouldn't block when trying to get + t = api.exc_after(0.1, api.TimeoutError) + self.assert_(pool.get()) + t.cancel() + finally: + sys.stderr = normal_err def test_track_events(self): pool = self.klass(track_events=True) @@ -70,6 +112,42 @@ class TestCoroutinePool(LimitedTestCase): return 'ok' pool.execute(slow) self.assertEquals(pool.wait(), 'ok') + + def test_pool_smash(self): + # The premise is that a coroutine in a Pool tries to get a token out + # of a token pool but times out before getting the token. We verify + # that neither pool is adversely affected by this situation. + from eventlet import pools + pool = self.klass(min_size=1, max_size=1) + tp = pools.TokenPool(max_size=1) + token = tp.get() # empty pool + def do_receive(tp): + api.exc_after(0, RuntimeError()) + try: + t = tp.get() + self.fail("Shouldn't have recieved anything from the pool") + except RuntimeError: + return 'timed out' + + # the execute makes the token pool expect that coroutine, but then + # immediately cuts bait + e1 = pool.execute(do_receive, tp) + self.assertEquals(e1.wait(), 'timed out') + + # the pool can get some random item back + def send_wakeup(tp): + tp.put('wakeup') + api.spawn(send_wakeup, tp) + + # now we ask the pool to run something else, which should not + # be affected by the previous send at all + def resume(): + return 'resumed' + e2 = pool.execute(resume) + self.assertEquals(e2.wait(), 'resumed') + + # we should be able to get out the thing we put in there, too + self.assertEquals(tp.get(), 'wakeup') class PoolBasicTests(LimitedTestCase): From 13b32b736c10b3a59be330c74d71be6e6aaaac2c Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 13 Jun 2009 13:41:28 -0700 Subject: [PATCH 18/23] Rewrote test_multiple_waiters to not also test the order of waiters returning. Added test_ordering_of_waiters that does test waiter ordering. --- greentest/test__coros_queue.py | 36 +++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/greentest/test__coros_queue.py b/greentest/test__coros_queue.py index 77939ac..2d672f2 100644 --- a/greentest/test__coros_queue.py +++ b/greentest/test__coros_queue.py @@ -74,6 +74,7 @@ class TestQueue(LimitedTestCase): self.assertEquals(e1.wait(),'done') def test_multiple_waiters(self): + # tests that multiple waiters get their results back q = coros.queue() def waiter(q, evt): @@ -86,14 +87,39 @@ class TestQueue(LimitedTestCase): api.sleep(0.01) # get 'em all waiting + results = set() + def collect_pending_results(): + for i, e in enumerate(evts): + timer = api.exc_after(0.001, api.TimeoutError) + try: + x = e.wait() + results.add(x) + timer.cancel() + except api.TimeoutError: + pass # no pending result at that event + return len(results) q.send(sendings[0]) - self.assertEquals(sendings[0], evts[0].wait()) + self.assertEquals(collect_pending_results(), 1) q.send(sendings[1]) - self.assertEquals(sendings[1], evts[1].wait()) + self.assertEquals(collect_pending_results(), 2) q.send(sendings[2]) q.send(sendings[3]) - self.assertEquals(sendings[2], evts[2].wait()) - self.assertEquals(sendings[3], evts[3].wait()) + self.assertEquals(collect_pending_results(), 4) + + def test_ordering_of_waiters(self): + # test that waiters receive results in the order that they waited + q = coros.queue() + def waiter(q, evt): + evt.send(q.wait()) + + sendings = list(range(10)) + evts = [coros.event() for x in sendings] + for i, x in enumerate(sendings): + api.spawn(waiter, q, evts[i]) + results = [] + for i, s in enumerate(sendings): + q.send(s) + self.assertEquals(s, evts[i].wait()) def test_waiters_that_cancel(self): q = coros.queue() @@ -175,7 +201,7 @@ class TestQueue(LimitedTestCase): api.sleep(0) self.assertEquals(1, waiting(q)) q.send('hi') - api.sleep(0) # *FIX this should not be necessary + api.sleep(0) # *FIX: should not be necessary self.assertEquals(0, waiting(q)) self.assertEquals('hi', e1.wait()) self.assertEquals(0, waiting(q)) From d47486377a2cca485c3da50d2ff2a6e43a34ff68 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 13 Jun 2009 14:05:59 -0700 Subject: [PATCH 19/23] Fixed bug in error method found by Denis. Thanks\! --- eventlet/httpd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eventlet/httpd.py b/eventlet/httpd.py index f7833b7..ae7a66c 100644 --- a/eventlet/httpd.py +++ b/eventlet/httpd.py @@ -322,7 +322,7 @@ class Request(object): self.write(body) return try: - produce(body, self) + self.site.adapt(body, self) except Exception, e: traceback.print_exc(file=self.log) if not self.response_written(): From 9035e92651b3ef7179db74be8c5f3865b34b7f3e Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 17 Jun 2009 17:20:05 -0700 Subject: [PATCH 20/23] Disabled __del__ doing a put back to connection pool (best practice has always been to do put in a finally block). Disabled two_simultaneous_connections test since it's timing-sensitive. Reenabled these tests in runall.py. --- eventlet/db_pool.py | 4 +++- greentest/db_pool_test.py | 22 ++++++++++++++++++---- greentest/runall.py | 2 +- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/eventlet/db_pool.py b/eventlet/db_pool.py index 5a8ea55..a1138bf 100644 --- a/eventlet/db_pool.py +++ b/eventlet/db_pool.py @@ -463,7 +463,9 @@ class PooledConnectionWrapper(GenericConnectionWrapper): self._destroy() def __del__(self): - self.close() + return # this causes some issues if __del__ is called in the + # main coroutine, so for now this is disabled + #self.close() class DatabaseConnector(object): diff --git a/greentest/db_pool_test.py b/greentest/db_pool_test.py index 2c2f57d..8518731 100644 --- a/greentest/db_pool_test.py +++ b/greentest/db_pool_test.py @@ -132,7 +132,9 @@ class TestDBConnectionPool(DBTester): self.assert_(self.pool.free() == 1) self.assertRaises(AttributeError, self.connection.cursor) - def test_deletion_does_a_put(self): + def dont_test_deletion_does_a_put(self): + # doing a put on del causes some issues if __del__ is called in the + # main coroutine, so, not doing that for now self.assert_(self.pool.free() == 0) self.connection = None self.assert_(self.pool.free() == 1) @@ -222,8 +224,9 @@ class TestDBConnectionPool(DBTester): conn.commit() - def test_two_simultaneous_connections(self): - """ This test is timing-sensitive. """ + def dont_test_two_simultaneous_connections(self): + # timing-sensitive test, disabled until we come up with a better + # way to do this self.pool = self.create_pool(2) conn = self.pool.get() self.set_up_test_table(conn) @@ -393,7 +396,7 @@ class TestDBConnectionPool(DBTester): self.assertEquals(self.pool.free(), 0) self.assertEquals(self.pool.waiting(), 1) self.pool.put(conn) - timer = api.exc_after(0.3, api.TimeoutError) + timer = api.exc_after(1, api.TimeoutError) conn = e.wait() timer.cancel() self.assertEquals(self.pool.free(), 0) @@ -442,6 +445,17 @@ class TestTpoolConnectionPool(TestDBConnectionPool): max_idle=max_idle, max_age=max_age, connect_timeout = connect_timeout, **self._auth) + + + def setUp(self): + from eventlet import tpool + tpool.QUIET = True + super(TestTpoolConnectionPool, self).setUp() + + def tearDown(self): + from eventlet import tpool + tpool.QUIET = False + super(TestTpoolConnectionPool, self).tearDown() class TestSaranwrapConnectionPool(TestDBConnectionPool): diff --git a/greentest/runall.py b/greentest/runall.py index f07b406..912466e 100755 --- a/greentest/runall.py +++ b/greentest/runall.py @@ -39,7 +39,7 @@ PARSE_PERIOD = 10 # the following aren't in the default list unless --all option present NOT_HUBS = set() NOT_REACTORS = set(['wxreactor', 'glib2reactor', 'gtk2reactor']) -NOT_TESTS = set(['db_pool_test.py']) +NOT_TESTS = set() def w(s): sys.stderr.write("%s\n" % (s, )) From 0031f52fd8e1ee6944e6f95d0c83a858dec4020e Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 17 Jun 2009 17:24:24 -0700 Subject: [PATCH 21/23] Added underlines on hover to html output so fools like me know they are links. --- greentest/generate_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/greentest/generate_report.py b/greentest/generate_report.py index 5014d7d..ff5ec7f 100755 --- a/greentest/generate_report.py +++ b/greentest/generate_report.py @@ -195,7 +195,7 @@ def format_header(rev, changeset, pyversion): return result def format_html(table, rev, changeset, pyversion): - r = '' + r = '' r += format_header(rev, changeset, pyversion) r += table r += '' From 9cd7b8fcf7ec424f6f4e9962dc86fae0e0e5d933 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 25 Jun 2009 17:51:37 -0700 Subject: [PATCH 22/23] Bug in declaration of channel class. --- eventlet/channel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eventlet/channel.py b/eventlet/channel.py index 80146e5..c431739 100644 --- a/eventlet/channel.py +++ b/eventlet/channel.py @@ -1,6 +1,6 @@ from eventlet import coros -class channel(coros.queue): +class channel(coros.Queue): def __init__(self): coros.queue.__init__(self, 0) From d4b669902ec4d585bbcfbc07e3577c7a23fe5dbb Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Fri, 26 Jun 2009 22:36:46 -0700 Subject: [PATCH 23/23] Described changes to httpd, saranwrap, db_pool --- NEWS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS b/NEWS index ceb8d5b..368f79b 100644 --- a/NEWS +++ b/NEWS @@ -45,6 +45,12 @@ The following classes are still present but will be removed in the future versio - channel.channel (use coros.Channel) - coros.CoroutinePool (use pool.Pool) +saranwrap.py now correctly closes the child process when the referring object is deleted, received some fixes to its detection of child process death, now correctly deals with the in keyword, and it is now possible to use coroutines in a non-blocking fashion in the child process. + +Time-based expiry added to db_pool. This adds the ability to expire connections both by idleness and also by total time open. There is also a connection timeout option. + +A small bug in httpd's error method was fixed. + Python 2.3 is no longer supported. A number of tests was added along with a script to run all of them for all the configurations.