From ce4435bdf34a825ce7ff930955d043cb2a58ec82 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 27 May 2009 00:43:51 -0700 Subject: [PATCH 01/21] Agh, sorry, we are using both httpdate and greenlib. --- eventlet/greenlib.py | 39 +++++++++++++++++++++++++++++++++++++++ eventlet/httpdate.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 eventlet/greenlib.py create mode 100644 eventlet/httpdate.py diff --git a/eventlet/greenlib.py b/eventlet/greenlib.py new file mode 100644 index 0000000..5f21783 --- /dev/null +++ b/eventlet/greenlib.py @@ -0,0 +1,39 @@ +"""\ +@file greenlib.py +@author Bob Ippolito + +Copyright (c) 2005-2006, Bob Ippolito +Copyright (c) 2007, Linden Research, Inc. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +def switch(other=None, value=None, exc=None): + """ + Switch to another greenlet, passing value or exception + """ + self = greenlet.getcurrent() + if other is None: + other = self.parent + if other is None: + other = self + if not (other or hasattr(other, 'run')): + raise SwitchingToDeadGreenlet("Switching to dead greenlet %r %r %r" % (other, value, exc)) + if exc: + return other.throw(exc) + else: + return other.switch(value) diff --git a/eventlet/httpdate.py b/eventlet/httpdate.py new file mode 100644 index 0000000..e1f81f3 --- /dev/null +++ b/eventlet/httpdate.py @@ -0,0 +1,39 @@ +"""\ +@file httpdate.py +@author Donovan Preston + +Copyright (c) 2006-2007, Linden Research, Inc. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import time + +__all__ = ['format_date_time'] + +# Weekday and month names for HTTP date/time formatting; always English! +_weekdayname = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] +_monthname = [None, # Dummy so we can use 1-based month numbers + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] + +def format_date_time(timestamp): + year, month, day, hh, mm, ss, wd, y, z = time.gmtime(timestamp) + return "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( + _weekdayname[wd], day, _monthname[month], year, hh, mm, ss + ) From f6681936df5ce22f316054f7a64aac1c308c4588 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 27 May 2009 01:11:30 -0700 Subject: [PATCH 02/21] Port of http://svn.secondlife.com/trac/eventlet/changeset/153 --- eventlet/httpc.py | 3 ++ greentest/httpc_test.py | 64 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/eventlet/httpc.py b/eventlet/httpc.py index b713a45..b868d9f 100644 --- a/eventlet/httpc.py +++ b/eventlet/httpc.py @@ -498,6 +498,9 @@ def make_connection(scheme, location, use_proxy): # run a little heuristic to see if location is an url, and if so parse out the hostpart if location.startswith('http'): _scheme, location, path, parameters, query, fragment = url_parser(location) + + if use_proxy and scheme == 'https': + scheme = 'http' result = scheme_to_factory_map[scheme](location) result.connect() diff --git a/greentest/httpc_test.py b/greentest/httpc_test.py index dbec99f..95110e4 100644 --- a/greentest/httpc_test.py +++ b/greentest/httpc_test.py @@ -400,5 +400,69 @@ class TestHttpTime(tests.TestCase): self.assertEqual(ticks, self.secs_since_epoch) +class TestProxy(tests.TestCase): + def test_ssl_proxy(self): + def ssl_proxy(sock): + conn, addr = sock.accept() + fd = conn.makefile() + try: + line = request = fd.readline() + self.assertEqual(request, 'GET https://localhost:1234 HTTP/1.1\r\n') + while line.strip(): # eat request headers + line = fd.readline() + + # we're not going to actually proxy to localhost:1234, + # we're just going to return a response on its behalf + fd.write("HTTP/1.0 200 OK\r\n\r\n") + finally: + fd.close() + conn.close() + + server = api.tcp_listener(('0.0.0.0', 5505)) + api.spawn(ssl_proxy, server) + import os + os.environ['ALL_PROXY'] = 'localhost:5505' + httpc.get('https://localhost:1234', ok=[200], use_proxy=True) + + def test_ssl_proxy_redirects(self): + # make sure that if the proxy returns a redirect, that httpc + # successfully follows it (this was broken at one point) + def ssl_proxy(sock): + conn, addr = sock.accept() + fd = conn.makefile() + try: + line = request = fd.readline() + self.assertEqual(request, 'GET https://localhost:1234 HTTP/1.1\r\n') + while line.strip(): # eat request headers + line = fd.readline() + + # we're not going to actually proxy to localhost:1234, + # we're just going to return a response on its behalf + fd.write("HTTP/1.0 302 Found\r\nLocation: https://localhost:1234/2\r\n\r\n") + finally: + fd.close() + conn.close() + + # second request, for /2 target + conn, addr = sock.accept() + fd = conn.makefile() + try: + line = request = fd.readline() + self.assertEqual(request, 'GET https://localhost:1234/2 HTTP/1.1\r\n') + while line.strip(): # eat request headers + line = fd.readline() + fd.write("HTTP/1.0 200 OK\r\n\r\n") + finally: + fd.close() + conn.close() + sock.close() + + server = api.tcp_listener(('0.0.0.0', 5505)) + api.spawn(ssl_proxy, server) + import os + os.environ['ALL_PROXY'] = 'localhost:5505' + httpc.get('https://localhost:1234', use_proxy=True, max_retries=1) + + if __name__ == '__main__': tests.main() From c2c5512f9e99a1e27112192fa1b037f780485ef2 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 27 May 2009 01:20:32 -0700 Subject: [PATCH 03/21] Port of http://svn.secondlife.com/trac/eventlet/changeset/157 --- eventlet/saranwrap.py | 37 +++++++++++++++++++++++++++++++++---- greentest/saranwrap_test.py | 7 +++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/eventlet/saranwrap.py b/eventlet/saranwrap.py index 07e3124..8af56f3 100644 --- a/eventlet/saranwrap.py +++ b/eventlet/saranwrap.py @@ -86,8 +86,8 @@ The wire protocol is to pickle the Request class in this file. The request class is basically an action and a map of parameters' """ -import os from cPickle import dumps, loads +import os import struct import sys @@ -182,6 +182,8 @@ class Request(object): def _read_lp_hunk(stream): len_bytes = stream.read(4) + if len_bytes == '': + raise EOFError("No more data to read from %s" % stream) length = struct.unpack('I', len_bytes)[0] body = stream.read(length) return body @@ -247,6 +249,7 @@ def _unmunge_attr_name(name): name = name[len('_Proxy'):] if(name.startswith('_ObjectProxy')): name = name[len('_ObjectProxy'):] + return name class ChildProcess(object): @@ -282,6 +285,9 @@ class ChildProcess(object): return retval + def __del__(self): + self._in.close() + class Proxy(object): """\ @@ -320,7 +326,10 @@ not supported, so you have to know what has been exported. request = Request('del', {'id':dead_object}) my_cp.make_request(request) - _dead_list.remove(dead_object) + try: + _dead_list.remove(dead_object) + except KeyError: + pass # Pass all public attributes across to find out if it is # callable or a simple attribute. @@ -423,6 +432,11 @@ not need to deal with this class directly.""" def proxied_type(self): + """ Returns the type of the object in the child process. + + Calling type(obj) on a saranwrapped object will always return + , so this is a way to get at the + 'real' type value.""" if type(self) is not ObjectProxy: return type(self) @@ -431,6 +445,14 @@ def proxied_type(self): request = Request('type', {'id':my_id}) return my_cp.make_request(request) + +def getpid(self): + """ Returns the pid of the child process. The argument should be + a saranwrapped object.""" + my_cp = self.__local_dict['_cp'] + return my_cp._in.getpid() + + class CallableProxy(object): """\ @class CallableProxy @@ -527,7 +549,11 @@ when the id is None.""" _log("del %s from %s" % (id, self._objects)) # *TODO what does __del__ actually return? - del self._objects[id] + try: + del self._objects[id] + except KeyError: + pass + return None def handle_type(self, obj, req): @@ -547,7 +573,10 @@ when the id is None.""" try: str_ = _read_lp_hunk(self._in) except EOFError: - sys.exit(0) # normal exit + if _g_debug_mode: + _log("Exiting normally") + sys.exit(0) + request = loads(str_) _log("request: %s (%s)" % (request, self._objects)) req = request diff --git a/greentest/saranwrap_test.py b/greentest/saranwrap_test.py index 7ac229b..0109c05 100644 --- a/greentest/saranwrap_test.py +++ b/greentest/saranwrap_test.py @@ -338,6 +338,13 @@ sys_path = sys.path""") 'random' in obj_proxy.get_dict(), 'Coroutine in saranwrapped object did not run') + def test_child_process_death(self): + prox = saranwrap.wrap({}) + pid = saranwrap.getpid(prox) + self.assertEqual(os.kill(pid, 0), None) # assert that the process is running + del prox # removing all references to the proxy should kill the child process + self.assertRaises(OSError, os.kill, pid, 0) # raises OSError if pid doesn't exist + def test_detection_of_server_crash(self): # make the server crash here pass From e17a3feeb90fe17c04996dd5de05be6cf684dbad Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 27 May 2009 02:37:15 -0700 Subject: [PATCH 04/21] Reenabled the queue tests. The semaphore now releases to waiters in the order that they started waiting. There's a FIX note in there because the queue should not need a sleep before the waiter count is correct. --- eventlet/coros.py | 17 +++++++++++++---- greentest/test__coros_queue.py | 5 +++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/eventlet/coros.py b/eventlet/coros.py index 66e0820..a41d449 100644 --- a/eventlet/coros.py +++ b/eventlet/coros.py @@ -262,6 +262,7 @@ class Semaphore(object): def __init__(self, count=0): self.counter = count + self._order = collections.deque() self._waiters = {} def __str__(self): @@ -280,6 +281,7 @@ class Semaphore(object): return False while self.counter<=0: self._waiters[api.getcurrent()] = None + self._order.append(api.getcurrent()) try: api.get_hub().switch() finally: @@ -293,14 +295,21 @@ class Semaphore(object): def release(self, blocking=True): # `blocking' parameter is for consistency with BoundedSemaphore and is ignored self.counter += 1 - if self._waiters: + if self._waiters or self._order: api.get_hub().schedule_call_global(0, self._do_acquire) return True def _do_acquire(self): - if self._waiters and self.counter>0: - waiter, _unused = self._waiters.popitem() - waiter.switch() + if (self._waiters or self._order) and self.counter>0: + waiter = None + while not waiter and self._order: + waiter = self._order.popleft() + if waiter not in self._waiters: + waiter = None + + if waiter is not None: + self._waiters.pop(waiter) + waiter.switch() def __exit__(self, typ, val, tb): self.release() diff --git a/greentest/test__coros_queue.py b/greentest/test__coros_queue.py index 7a1d616..77939ac 100644 --- a/greentest/test__coros_queue.py +++ b/greentest/test__coros_queue.py @@ -73,7 +73,7 @@ class TestQueue(LimitedTestCase): self.assertEquals(e2.wait(),'hi') self.assertEquals(e1.wait(),'done') - def skip_test_multiple_waiters(self): + def test_multiple_waiters(self): q = coros.queue() def waiter(q, evt): @@ -164,7 +164,7 @@ class TestQueue(LimitedTestCase): self.assertEquals(e2.wait(), 'timed out') self.assertEquals(q.wait(), 'sent') - def disable_test_waiting(self): + def test_waiting(self): def do_wait(q, evt): result = q.wait() evt.send(result) @@ -175,6 +175,7 @@ class TestQueue(LimitedTestCase): api.sleep(0) self.assertEquals(1, waiting(q)) q.send('hi') + api.sleep(0) # *FIX this should not be necessary self.assertEquals(0, waiting(q)) self.assertEquals('hi', e1.wait()) self.assertEquals(0, waiting(q)) From 6f0cb1573b5abc167529770dd6279e343ca7f511 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 28 May 2009 16:25:52 -0700 Subject: [PATCH 05/21] Committing broken ssl wsgi test, in hopes that it can be fixed. --- greentest/wsgi_test.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/greentest/wsgi_test.py b/greentest/wsgi_test.py index 0ea836f..95c75ec 100644 --- a/greentest/wsgi_test.py +++ b/greentest/wsgi_test.py @@ -23,6 +23,7 @@ THE SOFTWARE. """ import cgi +import os from eventlet import api from eventlet import wsgi @@ -264,5 +265,25 @@ class TestHttpd(tests.TestCase): chunklen = int(fd.readline(), 16) self.assert_(chunks > 1) + def test_012_ssl_server(self): + from eventlet import httpc + def wsgi_app(self, environ, start_response): + print "wsgi_app" + print environ['wsgi.input'] + return [environ['wsgi.input'].read()] + + certificate_file = os.path.join(os.path.dirname(__file__), 'test_server.crt') + private_key_file = os.path.join(os.path.dirname(__file__), 'test_server.key') + + sock = api.ssl_listener(('', 4201), certificate_file, private_key_file) + + api.spawn(wsgi.server, sock, wsgi_app) + + print "pre request" + result = httpc.post("https://localhost:4201/foo", "abc") + self.assertEquals(result, 'abc') + print "post request" + + if __name__ == '__main__': tests.main() From abda8d075b82e400059aa8e79b2f14934b113e27 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 28 May 2009 19:59:52 -0700 Subject: [PATCH 06/21] Fixed failing test by adding new makefile() method to GreenSSL. --- eventlet/greenio.py | 36 +++++++++++++++++++++++++++++++++++- eventlet/httpc.py | 3 +++ greentest/wsgi_test.py | 7 ++----- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/eventlet/greenio.py b/eventlet/greenio.py index 794f64b..1dccd86 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -531,10 +531,30 @@ class GreenPipe(GreenFile): self.fd.fd.flush() + +class RefCount(object): + """ Reference counting class only to be used with GreenSSL objects """ + def __init__(self): + self._count = 1 + + def increment(self): + self._count += 1 + + def decrement(self): + self._count -= 1 + assert self._count >= 0 + + def is_referenced(self): + return self._count > 0 + + class GreenSSL(GreenSocket): - def __init__(self, fd): + def __init__(self, fd, refcount = None): GreenSocket.__init__(self, fd) self.sock = self + self._refcount = refcount + if refcount is None: + self._refcount = RefCount() read = read @@ -550,7 +570,21 @@ class GreenSSL(GreenSocket): def issuer(self): return self.fd.issuer() + def dup(self): + raise NotImplemented("Dup not supported on SSL sockets") + def makefile(self, *args, **kw): + self._refcount.increment() + return GreenFile(type(self)(self.fd, refcount = self._refcount)) + + def close(self): + self._refcount.decrement() + if self._refcount.is_referenced(): + return + super(GreenSSL, self).close() + + + def socketpair(*args): one, two = socket.socketpair(*args) diff --git a/eventlet/httpc.py b/eventlet/httpc.py index b868d9f..9e8f9c3 100644 --- a/eventlet/httpc.py +++ b/eventlet/httpc.py @@ -586,6 +586,9 @@ class HttpSuite(object): def _get_response_body(self, params, connection): if connection is None: connection = connect(params.url, params.use_proxy) + # if we're creating a new connection we know the caller + # isn't going to reuse it + params.headers['connection'] = 'close' connection.request(params.method, params.path, params.body, params.headers) params.response = connection.getresponse() diff --git a/greentest/wsgi_test.py b/greentest/wsgi_test.py index 95c75ec..a1e8a95 100644 --- a/greentest/wsgi_test.py +++ b/greentest/wsgi_test.py @@ -267,9 +267,8 @@ class TestHttpd(tests.TestCase): def test_012_ssl_server(self): from eventlet import httpc - def wsgi_app(self, environ, start_response): - print "wsgi_app" - print environ['wsgi.input'] + def wsgi_app(environ, start_response): + start_response('200 OK', {}) return [environ['wsgi.input'].read()] certificate_file = os.path.join(os.path.dirname(__file__), 'test_server.crt') @@ -279,10 +278,8 @@ class TestHttpd(tests.TestCase): api.spawn(wsgi.server, sock, wsgi_app) - print "pre request" result = httpc.post("https://localhost:4201/foo", "abc") self.assertEquals(result, 'abc') - print "post request" if __name__ == '__main__': From e297081648879db27aeb2b52df6c04ffb450b117 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 2 Jun 2009 15:06:12 -0700 Subject: [PATCH 07/21] Test and fix for weird 'Unexpected EOF' error discovered through use. wsgi_test.py may not be the best place for this test but at least it's there. --- eventlet/greenio.py | 7 +++++++ greentest/wsgi_test.py | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/eventlet/greenio.py b/eventlet/greenio.py index 1dccd86..64508d3 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -557,6 +557,13 @@ class GreenSSL(GreenSocket): self._refcount = RefCount() read = read + + def sendall(self, data): + # overriding sendall because ssl sockets behave badly when asked to + # send empty strings; 'normal' sockets don't have a problem + if not data: + return + super(GreenSSL, self).sendall(data) def write(self, data): try: diff --git a/greentest/wsgi_test.py b/greentest/wsgi_test.py index a1e8a95..f896c93 100644 --- a/greentest/wsgi_test.py +++ b/greentest/wsgi_test.py @@ -1,5 +1,5 @@ """\ -@file httpd_test.py +@file wsgi_test.py @author Donovan Preston Copyright (c) 2007, Linden Research, Inc. @@ -280,6 +280,20 @@ class TestHttpd(tests.TestCase): result = httpc.post("https://localhost:4201/foo", "abc") self.assertEquals(result, 'abc') + + def test_013_empty_return(self): + from eventlet import httpc + def wsgi_app(environ, start_response): + start_response("200 OK", []) + return [""] + + certificate_file = os.path.join(os.path.dirname(__file__), 'test_server.crt') + private_key_file = os.path.join(os.path.dirname(__file__), 'test_server.key') + sock = api.ssl_listener(('', 4202), certificate_file, private_key_file) + api.spawn(wsgi.server, sock, wsgi_app) + + res = httpc.get("https://localhost:4202/foo") + self.assertEquals(res, '') if __name__ == '__main__': From 821e88debd82bd48b79624b5e9770efa250745ed Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 2 Jun 2009 17:11:48 -0700 Subject: [PATCH 08/21] Fixed test_connect_ssl --- eventlet/greenio.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eventlet/greenio.py b/eventlet/greenio.py index 64508d3..a9531b0 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -578,11 +578,13 @@ class GreenSSL(GreenSocket): return self.fd.issuer() def dup(self): - raise NotImplemented("Dup not supported on SSL sockets") + raise NotImplementedError("Dup not supported on SSL sockets") def makefile(self, *args, **kw): self._refcount.increment() return GreenFile(type(self)(self.fd, refcount = self._refcount)) + + makeGreenFile = makefile def close(self): self._refcount.decrement() From d396d974140201498bc19b6ab1667a19cef20c59 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 2 Jun 2009 17:50:24 -0700 Subject: [PATCH 09/21] Fixed timer canceling in CoroutinePool test, this mean that each coroutine in the pool is created anew. --- eventlet/pools.py | 11 +++-------- greentest/pools_test.py | 12 ++++++------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/eventlet/pools.py b/eventlet/pools.py index 0ddbd1d..18ee6f3 100644 --- a/eventlet/pools.py +++ b/eventlet/pools.py @@ -274,7 +274,9 @@ class CoroutinePool(Pool): def _main_loop(self, sender): """ Private, infinite loop run by a pooled coroutine. """ try: - while True: + # not really a loop anymore because we want the greenlet + # to die so all its timers get canceled + if True: recvd = sender.wait() # Delete the sender's result here because the very # first event through the loop is referenced by @@ -285,21 +287,14 @@ class CoroutinePool(Pool): # tests. sender._result = coros.NOT_USED - sender = coros.event() (evt, func, args, kw) = recvd self._safe_apply(evt, func, args, kw) - #api.get_hub().cancel_timers(api.getcurrent()) # Likewise, delete these variables or else they will # be referenced by this frame until replaced by the # next recvd, which may or may not be a long time from # now. del evt, func, args, kw, recvd - - self.put(sender) finally: - # if we get here, something broke badly, and all we can really - # do is try to keep the pool from leaking items. - # Shouldn't even try to print the exception. self.put(self.create()) def _safe_apply(self, evt, func, args, kw): diff --git a/greentest/pools_test.py b/greentest/pools_test.py index 3b70560..d81c8d1 100644 --- a/greentest/pools_test.py +++ b/greentest/pools_test.py @@ -284,16 +284,16 @@ class TestCoroutinePool(tests.TestCase): self.assertEquals(['cons1', 'prod', 'cons2'], results) def test_timer_cancel(self): + timer_fired = [] + def fire_timer(): + timer_fired.append(True) def some_work(): - t = timer.Timer(5, lambda: None) - t.autocancellable = True - t.schedule() - return t + api.get_hub().schedule_call_local(0, fire_timer) pool = pools.CoroutinePool(0, 2) worker = pool.execute(some_work) - t = worker.wait() + worker.wait() api.sleep(0) - self.assertEquals(t.cancelled, True) + self.assertEquals(timer_fired, []) def test_reentrant(self): pool = pools.CoroutinePool(0,1) From d8006b3a0523628236b76e7c1740e52bb0ab2d4d Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 2 Jun 2009 17:55:18 -0700 Subject: [PATCH 10/21] Fixed processes_test breakage, and removed mode='static' from that file. --- greentest/processes_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/greentest/processes_test.py b/greentest/processes_test.py index 98ce505..b5eee6a 100644 --- a/greentest/processes_test.py +++ b/greentest/processes_test.py @@ -21,12 +21,13 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import sys + from greentest import tests from eventlet import api from eventlet import processes class TestEchoPool(tests.TestCase): - mode = 'static' def setUp(self): self.pool = processes.ProcessPool('echo', ["hello"]) @@ -50,7 +51,6 @@ class TestEchoPool(tests.TestCase): class TestCatPool(tests.TestCase): - mode = 'static' def setUp(self): self.pool = processes.ProcessPool('cat') @@ -92,7 +92,6 @@ class TestCatPool(tests.TestCase): class TestDyingProcessesLeavePool(tests.TestCase): - mode = 'static' def setUp(self): self.pool = processes.ProcessPool('echo', ['hello'], max_size=1) @@ -112,11 +111,12 @@ class TestDyingProcessesLeavePool(tests.TestCase): class TestProcessLivesForever(tests.TestCase): - mode = 'static' def setUp(self): - self.pool = processes.ProcessPool('python', ['-c', 'print "y"; import time; time.sleep(0.1); print "y"'], max_size=1) + self.pool = processes.ProcessPool(sys.executable, ['-c', 'print "y"; import time; time.sleep(0.4); print "y"'], max_size=1) def test_reading_twice_from_same_process(self): + # this test is a little timing-sensitive in that if the sub-process + # completes its sleep before we do a full put/get then it will fail proc = self.pool.get() try: result = proc.read(2) @@ -125,7 +125,7 @@ class TestProcessLivesForever(tests.TestCase): self.pool.put(proc) proc2 = self.pool.get() - self.assert_(proc is proc2) + self.assert_(proc is proc2, "This will fail if there is a timing issue") try: result = proc2.read(2) self.assertEquals(result, 'y\n') From 06482faad69eec785451646621b9b3afaf4e431f Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 2 Jun 2009 18:23:49 -0700 Subject: [PATCH 11/21] Fixed test_saranwrap --- greentest/saranwrap_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/greentest/saranwrap_test.py b/greentest/saranwrap_test.py index 0109c05..d6ad3c8 100644 --- a/greentest/saranwrap_test.py +++ b/greentest/saranwrap_test.py @@ -343,6 +343,7 @@ sys_path = sys.path""") pid = saranwrap.getpid(prox) self.assertEqual(os.kill(pid, 0), None) # assert that the process is running del prox # removing all references to the proxy should kill the child process + api.sleep(0.1) # need to let the signal handler run self.assertRaises(OSError, os.kill, pid, 0) # raises OSError if pid doesn't exist def test_detection_of_server_crash(self): From 6a27fca3ca22c4effa8ffc14e3575e1317bf5d5a Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Fri, 5 Jun 2009 14:01:31 -0700 Subject: [PATCH 12/21] Minor fix to socket method so that it returns a real socket, in a kind of abstraction-violating way. --- eventlet/httpd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eventlet/httpd.py b/eventlet/httpd.py index cc5ff9d..0111c4c 100644 --- a/eventlet/httpd.py +++ b/eventlet/httpd.py @@ -311,7 +311,7 @@ class Request(object): return self._outgoing_headers.has_key(key.lower()) def socket(self): - return self.protocol.socket + return self.protocol.rfile._sock def error(self, response=None, body=None, log_traceback=True): if log_traceback: From 25b953bb38a2d5977dee65359dd4c0d197def972 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 7 Jun 2009 10:09:39 -0700 Subject: [PATCH 13/21] Slightly better error message when malformed urls make it into make_connection --- eventlet/httpc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eventlet/httpc.py b/eventlet/httpc.py index 9e8f9c3..125a29c 100644 --- a/eventlet/httpc.py +++ b/eventlet/httpc.py @@ -510,8 +510,10 @@ def make_connection(scheme, location, use_proxy): def connect(url, use_proxy=False): """ Create a connection object to the host specified in a url. Convenience function for make_connection.""" scheme, location = url_parser(url)[:2] - return make_connection(scheme, location, use_proxy) - + try: + return make_connection(scheme, location, use_proxy) + except KeyError: + raise ValueError("Unknown url scheme %s in url %s" % (scheme, url)) def make_safe_loader(loader): if not callable(loader): From 27932cdffcc90cace784095c742619d76855b06c Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 7 Jun 2009 22:10:27 -0700 Subject: [PATCH 14/21] Nat's patch for making greenio successfully import under Windows by deferring the import of fcntl. --- eventlet/greenio.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/eventlet/greenio.py b/eventlet/greenio.py index a9531b0..6baa94f 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -32,7 +32,6 @@ import errno import os import socket from socket import socket as _original_socket -import fcntl import time @@ -169,14 +168,17 @@ def file_send(fd, data): def set_nonblocking(fd): - ## Socket - if hasattr(fd, 'setblocking'): - fd.setblocking(0) - ## File - else: + try: + setblocking = fd.setblocking + except AttributeError: + # This version of Python predates socket.setblocking() + import fcntl fileno = fd.fileno() flags = fcntl.fcntl(fileno, fcntl.F_GETFL) fcntl.fcntl(fileno, fcntl.F_SETFL, flags | os.O_NONBLOCK) + else: + # socket supports setblocking() + setblocking(0) class GreenSocket(object): From 4d7cb26915d1ee41e9a738b1abffd83b25a1ad9f Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 13 Jun 2009 13:28:24 -0700 Subject: [PATCH 15/21] Weird encoding bug in generate_report. --- greentest/generate_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/greentest/generate_report.py b/greentest/generate_report.py index f2449ef..5014d7d 100755 --- a/greentest/generate_report.py +++ b/greentest/generate_report.py @@ -205,7 +205,7 @@ def generate_raw_results(path, database): c = sqlite3.connect(database) res = c.execute('select id, stdout from command_record').fetchall() for id, out in res: - file(os.path.join(path, '%s.txt' % id), 'w').write(out) + file(os.path.join(path, '%s.txt' % id), 'w').write(out.encode('utf-8')) sys.stderr.write('.') sys.stderr.write('\n') From 740c39503c4237681b5ed2e0d0fc31dfedb7bf35 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 13 Jun 2009 13:38:23 -0700 Subject: [PATCH 16/21] Copyright header merge. --- eventlet/greenlib.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/eventlet/greenlib.py b/eventlet/greenlib.py index c1babb6..8de50e0 100644 --- a/eventlet/greenlib.py +++ b/eventlet/greenlib.py @@ -1,24 +1,23 @@ -"""\ -Copyright (c) 2005-2006, Bob Ippolito -Copyright (c) 2007, Linden Research, Inc. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +# Copyright (c) 2005-2006, Bob Ippolito +# Copyright (c) 2007, Linden Research, Inc. +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -""" from eventlet.api import Greenlet class SwitchingToDeadGreenlet(Exception): From 728f79aab583ca43b285ba97c69e12ea287b4703 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 13 Jun 2009 13:39:15 -0700 Subject: [PATCH 17/21] Re-added relevant tests to test__pool.py. They both fail, and shouldn't --- greentest/test__pool.py | 78 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/greentest/test__pool.py b/greentest/test__pool.py index 38354c7..2b5e093 100644 --- a/greentest/test__pool.py +++ b/greentest/test__pool.py @@ -39,6 +39,20 @@ class TestCoroutinePool(LimitedTestCase): done.wait() self.assertEquals(['cons1', 'prod', 'cons2'], results) + def test_timer_cancel(self): + # this test verifies that local timers are not fired + # outside of the context of the execute method + timer_fired = [] + def fire_timer(): + timer_fired.append(True) + def some_work(): + api.get_hub().schedule_call_local(0, fire_timer) + pool = self.klass(0, 2) + worker = pool.execute(some_work) + worker.wait() + api.sleep(0) + self.assertEquals(timer_fired, []) + def test_reentrant(self): pool = self.klass(0,1) def reenter(): @@ -55,6 +69,34 @@ class TestCoroutinePool(LimitedTestCase): pool.execute_async(reenter_async) evt.wait() + + def test_stderr_raising(self): + # testing that really egregious errors in the error handling code + # (that prints tracebacks to stderr) don't cause the pool to lose + # any members + import sys + pool = self.klass(min_size=1, max_size=1) + def crash(*args, **kw): + raise RuntimeError("Whoa") + class FakeFile(object): + write = crash + + # we're going to do this by causing the traceback.print_exc in + # safe_apply to raise an exception and thus exit _main_loop + normal_err = sys.stderr + try: + sys.stderr = FakeFile() + waiter = pool.execute(crash) + self.assertRaises(RuntimeError, waiter.wait) + # the pool should have something free at this point since the + # waiter returned + self.assertEqual(pool.free(), 1) + # shouldn't block when trying to get + t = api.exc_after(0.1, api.TimeoutError) + self.assert_(pool.get()) + t.cancel() + finally: + sys.stderr = normal_err def test_track_events(self): pool = self.klass(track_events=True) @@ -70,6 +112,42 @@ class TestCoroutinePool(LimitedTestCase): return 'ok' pool.execute(slow) self.assertEquals(pool.wait(), 'ok') + + def test_pool_smash(self): + # The premise is that a coroutine in a Pool tries to get a token out + # of a token pool but times out before getting the token. We verify + # that neither pool is adversely affected by this situation. + from eventlet import pools + pool = self.klass(min_size=1, max_size=1) + tp = pools.TokenPool(max_size=1) + token = tp.get() # empty pool + def do_receive(tp): + api.exc_after(0, RuntimeError()) + try: + t = tp.get() + self.fail("Shouldn't have recieved anything from the pool") + except RuntimeError: + return 'timed out' + + # the execute makes the token pool expect that coroutine, but then + # immediately cuts bait + e1 = pool.execute(do_receive, tp) + self.assertEquals(e1.wait(), 'timed out') + + # the pool can get some random item back + def send_wakeup(tp): + tp.put('wakeup') + api.spawn(send_wakeup, tp) + + # now we ask the pool to run something else, which should not + # be affected by the previous send at all + def resume(): + return 'resumed' + e2 = pool.execute(resume) + self.assertEquals(e2.wait(), 'resumed') + + # we should be able to get out the thing we put in there, too + self.assertEquals(tp.get(), 'wakeup') class PoolBasicTests(LimitedTestCase): From 13b32b736c10b3a59be330c74d71be6e6aaaac2c Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 13 Jun 2009 13:41:28 -0700 Subject: [PATCH 18/21] Rewrote test_multiple_waiters to not also test the order of waiters returning. Added test_ordering_of_waiters that does test waiter ordering. --- greentest/test__coros_queue.py | 36 +++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/greentest/test__coros_queue.py b/greentest/test__coros_queue.py index 77939ac..2d672f2 100644 --- a/greentest/test__coros_queue.py +++ b/greentest/test__coros_queue.py @@ -74,6 +74,7 @@ class TestQueue(LimitedTestCase): self.assertEquals(e1.wait(),'done') def test_multiple_waiters(self): + # tests that multiple waiters get their results back q = coros.queue() def waiter(q, evt): @@ -86,14 +87,39 @@ class TestQueue(LimitedTestCase): api.sleep(0.01) # get 'em all waiting + results = set() + def collect_pending_results(): + for i, e in enumerate(evts): + timer = api.exc_after(0.001, api.TimeoutError) + try: + x = e.wait() + results.add(x) + timer.cancel() + except api.TimeoutError: + pass # no pending result at that event + return len(results) q.send(sendings[0]) - self.assertEquals(sendings[0], evts[0].wait()) + self.assertEquals(collect_pending_results(), 1) q.send(sendings[1]) - self.assertEquals(sendings[1], evts[1].wait()) + self.assertEquals(collect_pending_results(), 2) q.send(sendings[2]) q.send(sendings[3]) - self.assertEquals(sendings[2], evts[2].wait()) - self.assertEquals(sendings[3], evts[3].wait()) + self.assertEquals(collect_pending_results(), 4) + + def test_ordering_of_waiters(self): + # test that waiters receive results in the order that they waited + q = coros.queue() + def waiter(q, evt): + evt.send(q.wait()) + + sendings = list(range(10)) + evts = [coros.event() for x in sendings] + for i, x in enumerate(sendings): + api.spawn(waiter, q, evts[i]) + results = [] + for i, s in enumerate(sendings): + q.send(s) + self.assertEquals(s, evts[i].wait()) def test_waiters_that_cancel(self): q = coros.queue() @@ -175,7 +201,7 @@ class TestQueue(LimitedTestCase): api.sleep(0) self.assertEquals(1, waiting(q)) q.send('hi') - api.sleep(0) # *FIX this should not be necessary + api.sleep(0) # *FIX: should not be necessary self.assertEquals(0, waiting(q)) self.assertEquals('hi', e1.wait()) self.assertEquals(0, waiting(q)) From d47486377a2cca485c3da50d2ff2a6e43a34ff68 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 13 Jun 2009 14:05:59 -0700 Subject: [PATCH 19/21] Fixed bug in error method found by Denis. Thanks\! --- eventlet/httpd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eventlet/httpd.py b/eventlet/httpd.py index f7833b7..ae7a66c 100644 --- a/eventlet/httpd.py +++ b/eventlet/httpd.py @@ -322,7 +322,7 @@ class Request(object): self.write(body) return try: - produce(body, self) + self.site.adapt(body, self) except Exception, e: traceback.print_exc(file=self.log) if not self.response_written(): From 9035e92651b3ef7179db74be8c5f3865b34b7f3e Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 17 Jun 2009 17:20:05 -0700 Subject: [PATCH 20/21] Disabled __del__ doing a put back to connection pool (best practice has always been to do put in a finally block). Disabled two_simultaneous_connections test since it's timing-sensitive. Reenabled these tests in runall.py. --- eventlet/db_pool.py | 4 +++- greentest/db_pool_test.py | 22 ++++++++++++++++++---- greentest/runall.py | 2 +- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/eventlet/db_pool.py b/eventlet/db_pool.py index 5a8ea55..a1138bf 100644 --- a/eventlet/db_pool.py +++ b/eventlet/db_pool.py @@ -463,7 +463,9 @@ class PooledConnectionWrapper(GenericConnectionWrapper): self._destroy() def __del__(self): - self.close() + return # this causes some issues if __del__ is called in the + # main coroutine, so for now this is disabled + #self.close() class DatabaseConnector(object): diff --git a/greentest/db_pool_test.py b/greentest/db_pool_test.py index 2c2f57d..8518731 100644 --- a/greentest/db_pool_test.py +++ b/greentest/db_pool_test.py @@ -132,7 +132,9 @@ class TestDBConnectionPool(DBTester): self.assert_(self.pool.free() == 1) self.assertRaises(AttributeError, self.connection.cursor) - def test_deletion_does_a_put(self): + def dont_test_deletion_does_a_put(self): + # doing a put on del causes some issues if __del__ is called in the + # main coroutine, so, not doing that for now self.assert_(self.pool.free() == 0) self.connection = None self.assert_(self.pool.free() == 1) @@ -222,8 +224,9 @@ class TestDBConnectionPool(DBTester): conn.commit() - def test_two_simultaneous_connections(self): - """ This test is timing-sensitive. """ + def dont_test_two_simultaneous_connections(self): + # timing-sensitive test, disabled until we come up with a better + # way to do this self.pool = self.create_pool(2) conn = self.pool.get() self.set_up_test_table(conn) @@ -393,7 +396,7 @@ class TestDBConnectionPool(DBTester): self.assertEquals(self.pool.free(), 0) self.assertEquals(self.pool.waiting(), 1) self.pool.put(conn) - timer = api.exc_after(0.3, api.TimeoutError) + timer = api.exc_after(1, api.TimeoutError) conn = e.wait() timer.cancel() self.assertEquals(self.pool.free(), 0) @@ -442,6 +445,17 @@ class TestTpoolConnectionPool(TestDBConnectionPool): max_idle=max_idle, max_age=max_age, connect_timeout = connect_timeout, **self._auth) + + + def setUp(self): + from eventlet import tpool + tpool.QUIET = True + super(TestTpoolConnectionPool, self).setUp() + + def tearDown(self): + from eventlet import tpool + tpool.QUIET = False + super(TestTpoolConnectionPool, self).tearDown() class TestSaranwrapConnectionPool(TestDBConnectionPool): diff --git a/greentest/runall.py b/greentest/runall.py index f07b406..912466e 100755 --- a/greentest/runall.py +++ b/greentest/runall.py @@ -39,7 +39,7 @@ PARSE_PERIOD = 10 # the following aren't in the default list unless --all option present NOT_HUBS = set() NOT_REACTORS = set(['wxreactor', 'glib2reactor', 'gtk2reactor']) -NOT_TESTS = set(['db_pool_test.py']) +NOT_TESTS = set() def w(s): sys.stderr.write("%s\n" % (s, )) From 0031f52fd8e1ee6944e6f95d0c83a858dec4020e Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 17 Jun 2009 17:24:24 -0700 Subject: [PATCH 21/21] Added underlines on hover to html output so fools like me know they are links. --- greentest/generate_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/greentest/generate_report.py b/greentest/generate_report.py index 5014d7d..ff5ec7f 100755 --- a/greentest/generate_report.py +++ b/greentest/generate_report.py @@ -195,7 +195,7 @@ def format_header(rev, changeset, pyversion): return result def format_html(table, rev, changeset, pyversion): - r = '' + r = '' r += format_header(rev, changeset, pyversion) r += table r += ''