From c18067387c603100bd8bb22c4819155332ac584e Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 5 Dec 2009 00:49:00 -0800 Subject: [PATCH 01/14] Improved GreenSocket construction, added connect timeout test. --- eventlet/greenio.py | 5 +++++ tests/greenio_test.py | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/eventlet/greenio.py b/eventlet/greenio.py index 9e0ec06..982dd1b 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -167,6 +167,7 @@ class GreenSocket(object): fd = family_or_realsock assert not args, args assert not kwargs, kwargs + orig_timeout = fd.gettimeout() set_nonblocking(fd) self.fd = fd @@ -181,6 +182,10 @@ class GreenSocket(object): # act non-blocking self.act_non_blocking = False + # import timeout from the other fd if it's distinct + if orig_timeout and orig_timeout is not self.timeout: + self.settimeout(orig_timeout) + @property def _sock(self): return self diff --git a/tests/greenio_test.py b/tests/greenio_test.py index 28ebde6..cbe6b4c 100644 --- a/tests/greenio_test.py +++ b/tests/greenio_test.py @@ -29,6 +29,12 @@ def min_buf_size(): return test_sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) class TestGreenIo(LimitedTestCase): + def test_connect_timeout(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(0.1) + gs = greenio.GreenSocket(s) + self.assertRaises(socket.timeout, gs.connect, ('192.0.2.1', 80)) + def test_close_with_makefile(self): def accept_close_early(listener): # verify that the makefile and the socket are truly independent From 1a5ce5758726e45944fd54c353fb7c2e12b98906 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 5 Dec 2009 00:51:15 -0800 Subject: [PATCH 02/14] Arg, stupid inheritance --- eventlet/greenio.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eventlet/greenio.py b/eventlet/greenio.py index 982dd1b..2aa6d67 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -167,7 +167,10 @@ class GreenSocket(object): fd = family_or_realsock assert not args, args assert not kwargs, kwargs - orig_timeout = fd.gettimeout() + try: + orig_timeout = fd.gettimeout() + except AttributeError: + orig_timeout = None set_nonblocking(fd) self.fd = fd From def5bb8e8f318f55a08a7f516b2198e73bcd5beb Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 5 Dec 2009 13:27:55 -0800 Subject: [PATCH 03/14] Renamed GreenFile to Green_fileobject, better reflecting its purpose. This is kind of a documentation fix for #6. --- eventlet/greenio.py | 11 ++++++----- eventlet/tpool.py | 2 +- tests/greenio_test.py | 8 +++++++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/eventlet/greenio.py b/eventlet/greenio.py index 2aa6d67..7585fef 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio.py @@ -14,7 +14,7 @@ import warnings from errno import EWOULDBLOCK, EAGAIN -__all__ = ['GreenSocket', 'GreenFile', 'GreenPipe'] +__all__ = ['GreenSocket', 'GreenPipe'] def higher_order_recv(recv_func): def recv(self, buflen, flags=0): @@ -302,7 +302,7 @@ class GreenSocket(object): return socket._fileobject(self.dup(), mode, bufsize) def makeGreenFile(self, mode='r', bufsize=-1): - return GreenFile(self.dup()) + return Green_fileobject(self.dup()) recv = higher_order_recv(socket_recv) @@ -370,8 +370,9 @@ class GreenSocket(object): return self.timeout - -class GreenFile(object): +class Green_fileobject(object): + """Green version of socket._fileobject, for use only with regular + sockets.""" newlines = '\r\n' mode = 'wb+' @@ -494,7 +495,7 @@ class GreenPipeSocket(GreenSocket): send = higher_order_send(file_send) -class GreenPipe(GreenFile): +class GreenPipe(Green_fileobject): def __init__(self, fd): set_nonblocking(fd) self.fd = GreenPipeSocket(fd) diff --git a/eventlet/tpool.py b/eventlet/tpool.py index 83bffc8..9c55774 100644 --- a/eventlet/tpool.py +++ b/eventlet/tpool.py @@ -200,7 +200,7 @@ def setup(): csock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) nsock, addr = sock.accept() nsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - _rfile = greenio.GreenFile(greenio.GreenSocket(csock)) + _rfile = greenio.Green_fileobject(greenio.GreenSocket(csock)) _wfile = nsock.makefile() for i in range(0,_nthreads): diff --git a/tests/greenio_test.py b/tests/greenio_test.py index cbe6b4c..bbe3563 100644 --- a/tests/greenio_test.py +++ b/tests/greenio_test.py @@ -2,6 +2,7 @@ from tests import skipped, LimitedTestCase, skip_with_libevent, TestIsTakingTooL from unittest import main from eventlet import api, util, coros, proc, greenio from eventlet.green.socket import GreenSSLObject +import errno import os import socket import sys @@ -33,7 +34,12 @@ class TestGreenIo(LimitedTestCase): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(0.1) gs = greenio.GreenSocket(s) - self.assertRaises(socket.timeout, gs.connect, ('192.0.2.1', 80)) + try: + self.assertRaises(socket.timeout, gs.connect, ('192.0.2.1', 80)) + except socket.error, e: + # unreachable is also a valid outcome + if e[0] != errno.EHOSTUNREACH: + raise def test_close_with_makefile(self): def accept_close_early(listener): From 94edc04921215f11a6b13b8ca7acb9aaffd6777b Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 5 Dec 2009 22:48:18 -0800 Subject: [PATCH 04/14] 0.9.1 branding --- NEWS | 4 +++- doc/real_index.html | 2 +- eventlet/__init__.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index d74e86d..a2479bb 100644 --- a/NEWS +++ b/NEWS @@ -13,7 +13,9 @@ * Added support for logging x-forwarded-for header in wsgi. * api.tcp_server is now deprecated, will be removed in a future release. * Added instructions on how to generate coverage reports to the documentation. -* Bug fixes in: wsgi.py, twistedr.py, poll.py, greenio.py, util.py, select.py, processes.py +* Renamed GreenFile to Green_fileobject, to better reflect its purpose. +* Deprecated erpc method in tpool.py +* Bug fixes in: wsgi.py, twistedr.py, poll.py, greenio.py, util.py, select.py, processes.py, selects.py 0.9.0 ===== diff --git a/doc/real_index.html b/doc/real_index.html index cf0e610..ac757cb 100644 --- a/doc/real_index.html +++ b/doc/real_index.html @@ -35,7 +35,7 @@ easy_install eventlet

Alternately, you can download the source tarball:

diff --git a/eventlet/__init__.py b/eventlet/__init__.py index 9e6e269..1e98450 100644 --- a/eventlet/__init__.py +++ b/eventlet/__init__.py @@ -1,2 +1,2 @@ -version_info = (0, 9, '1pre') +version_info = (0, 9, 1) __version__ = '%s.%s.%s' % version_info From 5db1ed5f590a5038ecd43ec89fd770e2be049509 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 6 Dec 2009 00:01:02 -0800 Subject: [PATCH 06/14] Fixed performance issue when apps yield tremendous quantities of tiny strings. --- AUTHORS | 2 +- eventlet/wsgi.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 9af4d75..0ffb0c6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -26,7 +26,7 @@ Thanks To * Chuck Thier, reporting a bug in processes.py * Brantley Harris, reporting bug #4 * Taso Du Val, reproing an exception squelching bug, saving children's lives ;-) -* R. Tyler Ballance, bug report on tpool on Windows, help with improving corolocal module, keen eye for redundancy +* R. Tyler Ballance, bug report on tpool on Windows, help with improving corolocal module, keen eye for redundancy, profile performance report * Sergey Shepelev, PEP 8 police :-), reporting bug #5 * Luci Stanescu, for reporting twisted hub bug * Marcus Cavanaugh, for test case code that has been incredibly useful in tracking down bugs diff --git a/eventlet/wsgi.py b/eventlet/wsgi.py index 9ae5c84..d4e7e69 100644 --- a/eventlet/wsgi.py +++ b/eventlet/wsgi.py @@ -261,11 +261,14 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler): 'Content-Length' not in [h for h, v in headers_set[1]]: headers_set[1].append(('Content-Length', str(sum(map(len, result))))) towrite = [] + towrite_size = 0 for data in result: towrite.append(data) - if sum(map(len, towrite)) >= self.minimum_chunk_size: + towrite_size += len(data) + if towrite_size >= self.minimum_chunk_size: write(''.join(towrite)) towrite = [] + towrite_size = 0 if towrite: write(''.join(towrite)) if not headers_sent or use_chunked[0]: From 15d4bc723bdad201095848ba5470922f83331612 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 6 Dec 2009 19:20:45 -0800 Subject: [PATCH 07/14] Finally, some peace and quiet around here. :-) --- eventlet/hubs/hub.py | 6 ++++-- tests/__init__.py | 14 ++++++++++++++ tests/coros_test.py | 20 ++++++-------------- tests/test__hub.py | 3 ++- tests/test__proc.py | 13 +++++++------ 5 files changed, 33 insertions(+), 23 deletions(-) diff --git a/eventlet/hubs/hub.py b/eventlet/hubs/hub.py index 0f8dade..250d80c 100644 --- a/eventlet/hubs/hub.py +++ b/eventlet/hubs/hub.py @@ -61,6 +61,7 @@ class BaseHub(object): 'exit': [], } self.lclass = FdListener + self.silent_timer_exceptions = False def add(self, evtype, fileno, cb): """ Signals an intent to or write a particular file descriptor. @@ -220,8 +221,9 @@ class BaseHub(object): self.squelch_observer_exception(observer, sys.exc_info()) def squelch_timer_exception(self, timer, exc_info): - traceback.print_exception(*exc_info) - print >>sys.stderr, "Timer raised: %r" % (timer,) + if not self.silent_timer_exceptions: + traceback.print_exception(*exc_info) + print >>sys.stderr, "Timer raised: %r" % (timer,) def _add_absolute_timer(self, when, info): # the 0 placeholder makes it easy to bisect_right using (now, 1) diff --git a/tests/__init__.py b/tests/__init__.py index c7e6a93..d0b6e04 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -82,6 +82,20 @@ class LimitedTestCase(unittest.TestCase): self.timer.cancel() +class SilencedTestCase(LimitedTestCase): + """ Subclass of LimitedTestCase that also silences the printing of timer + exceptions.""" + def setUp(self): + from eventlet import api + super(SilencedTestCase, self).setUp() + api.get_hub().silent_timer_exceptions = True + + def tearDown(self): + from eventlet import api + super(SilencedTestCase, self).tearDown() + api.get_hub().silent_timer_exceptions = False + + def find_command(command): for dir in os.getenv('PATH', '/usr/bin:/usr/sbin').split(os.pathsep): p = os.path.join(dir, command) diff --git a/tests/coros_test.py b/tests/coros_test.py index fc62420..4e630e8 100644 --- a/tests/coros_test.py +++ b/tests/coros_test.py @@ -1,15 +1,8 @@ -from unittest import TestCase, main +from unittest import main, TestCase +from tests import SilencedTestCase from eventlet import coros, api -class TestEvent(TestCase): - mode = 'static' - def setUp(self): - # raise an exception if we're waiting forever - self._cancel_timeout = api.exc_after(1, RuntimeError('test takes too long')) - - def tearDown(self): - self._cancel_timeout.cancel() - +class TestEvent(SilencedTestCase): def test_waiting_for_event(self): evt = coros.event() value = 'some stuff' @@ -81,15 +74,14 @@ class IncrActor(coros.Actor): if evt: evt.send() -class TestActor(TestCase): +class TestActor(SilencedTestCase): mode = 'static' def setUp(self): - # raise an exception if we're waiting forever - self._cancel_timeout = api.exc_after(1, api.TimeoutError()) + super(TestActor, self).setUp() self.actor = IncrActor() def tearDown(self): - self._cancel_timeout.cancel() + super(TestActor, self).tearDown() api.kill(self.actor._killer) def test_cast(self): diff --git a/tests/test__hub.py b/tests/test__hub.py index 631c3ee..007ce55 100644 --- a/tests/test__hub.py +++ b/tests/test__hub.py @@ -1,4 +1,5 @@ import unittest +from tests import SilencedTestCase import time from eventlet import api from eventlet.green import socket @@ -29,7 +30,7 @@ class TestDebug(unittest.TestCase): self.assert_(not api.get_hub().debug) -class TestExceptionInMainloop(unittest.TestCase): +class TestExceptionInMainloop(SilencedTestCase): def test_sleep(self): # even if there was an error in the mainloop, the hub should continue to work diff --git a/tests/test__proc.py b/tests/test__proc.py index ce1c1a0..aef4456 100644 --- a/tests/test__proc.py +++ b/tests/test__proc.py @@ -2,14 +2,14 @@ import sys import unittest from eventlet.api import sleep, with_timeout from eventlet import api, proc, coros -from tests import LimitedTestCase, skipped +from tests import SilencedTestCase, skipped DELAY = 0.01 class ExpectedError(Exception): pass -class TestLink_Signal(LimitedTestCase): +class TestLink_Signal(SilencedTestCase): def test_send(self): s = proc.Source() @@ -48,7 +48,7 @@ class TestLink_Signal(LimitedTestCase): self.assertRaises(OSError, s.wait) -class TestProc(LimitedTestCase): +class TestProc(SilencedTestCase): def test_proc(self): p = proc.spawn(lambda : 100) @@ -76,13 +76,13 @@ class TestProc(LimitedTestCase): self.assertRaises(proc.LinkedCompleted, sleep, 0.1) -class TestCase(LimitedTestCase): +class TestCase(SilencedTestCase): def link(self, p, listener=None): getattr(p, self.link_method)(listener) def tearDown(self): - LimitedTestCase.tearDown(self) + SilencedTestCase.tearDown(self) self.p.unlink() def set_links(self, p, first_time, kill_exc_type): @@ -252,7 +252,7 @@ class TestRaise_link_exception(TestRaise_link): link_method = 'link_exception' -class TestStuff(unittest.TestCase): +class TestStuff(SilencedTestCase): def test_wait_noerrors(self): x = proc.spawn(lambda : 1) @@ -297,6 +297,7 @@ class TestStuff(unittest.TestCase): proc.waitall([a, b]) except ExpectedError, ex: assert 'second' in str(ex), repr(str(ex)) + api.sleep(0.2) # sleep to ensure that the other timer is raised def test_multiple_listeners_error(self): # if there was an error while calling a callback From deed5bfd83264316bc0a734616f1b98fcdbf19de Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 6 Dec 2009 21:49:39 -0800 Subject: [PATCH 08/14] Corrected timeout behavior of ssl sockets, better docs on the annoying close behavior. --- eventlet/green/ssl.py | 37 ++++++++++++++++++++++++------------- eventlet/hubs/poll.py | 2 +- tests/stdlib/test_ssl.py | 6 +++--- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/eventlet/green/ssl.py b/eventlet/green/ssl.py index de006fe..9bdedec 100644 --- a/eventlet/green/ssl.py +++ b/eventlet/green/ssl.py @@ -11,12 +11,23 @@ from thread import get_ident from eventlet.greenio import set_nonblocking, GreenSocket, SOCKET_CLOSED, CONNECT_ERR, CONNECT_SUCCESS orig_socket = __import__('socket') socket = orig_socket.socket +timeout_exc = orig_socket.timeout class GreenSSLSocket(__ssl.SSLSocket): """ This is a green version of the SSLSocket class from the ssl module added in 2.6. For documentation on it, please see the Python standard - documentation.""" + documentation. + + Python nonblocking ssl objects don't give errors when the other end + of the socket is closed (they do notice when the other end is shutdown, + though). Any write/read operations will simply hang if the socket is + closed from the other end. There is no obvious fix for this problem; + it appears to be a limitation of Python's ssl object implementation. + A workaround is to set a reasonable timeout on the socket using + settimeout(), and to close/reopen the connection when a timeout + occurs at an unexpected juncture in the code. + """ # we are inheriting from SSLSocket because its constructor calls # do_handshake whose behavior we wish to override def __init__(self, sock, *args, **kw): @@ -62,12 +73,12 @@ class GreenSSLSocket(__ssl.SSLSocket): trampoline(self.fileno(), read=True, timeout=self.gettimeout(), - timeout_exc=SSLError) + timeout_exc=timeout_exc('timed out')) elif exc[0] == SSL_ERROR_WANT_WRITE: trampoline(self.fileno(), write=True, timeout=self.gettimeout(), - timeout_exc=SSLError) + timeout_exc=timeout_exc('timed out')) else: raise @@ -121,7 +132,7 @@ class GreenSSLSocket(__ssl.SSLSocket): raise ValueError("sendto not allowed on instances of %s" % self.__class__) else: - trampoline(self.fileno(), write=True, timeout_exc=orig_socket.timeout) + trampoline(self.fileno(), write=True, timeout_exc=timeout_exc('timed out')) return socket.sendto(self, data, addr, flags) def sendall (self, data, flags=0): @@ -146,7 +157,7 @@ class GreenSSLSocket(__ssl.SSLSocket): raise if e[0] == errno.EWOULDBLOCK: trampoline(self.fileno(), write=True, - timeout=self.gettimeout(), timeout_exc=orig_socket.timeout) + timeout=self.gettimeout(), timeout_exc=timeout_exc('timed out')) if e[0] in SOCKET_CLOSED: return '' raise @@ -169,7 +180,7 @@ class GreenSSLSocket(__ssl.SSLSocket): raise if e[0] == errno.EWOULDBLOCK: trampoline(self.fileno(), read=True, - timeout=self.gettimeout(), timeout_exc=orig_socket.timeout) + timeout=self.gettimeout(), timeout_exc=timeout_exc('timed out')) if e[0] in SOCKET_CLOSED: return '' raise @@ -177,17 +188,17 @@ class GreenSSLSocket(__ssl.SSLSocket): def recv_into (self, buffer, nbytes=None, flags=0): if not self.act_non_blocking: - trampoline(self.fileno(), read=True, timeout=self.gettimeout(), timeout_exc=orig_socket.timeout) + trampoline(self.fileno(), read=True, timeout=self.gettimeout(), timeout_exc=timeout_exc('timed out')) return super(GreenSSLSocket, self).recv_into(buffer, nbytes, flags) def recvfrom (self, addr, buflen=1024, flags=0): if not self.act_non_blocking: - trampoline(self.fileno(), read=True, timeout=self.gettimeout(), timeout_exc=orig_socket.timeout) + trampoline(self.fileno(), read=True, timeout=self.gettimeout(), timeout_exc=timeout_exc('timed out')) return super(GreenSSLSocket, self).recvfrom(addr, buflen, flags) def recvfrom_into (self, buffer, nbytes=None, flags=0): if not self.act_non_blocking: - trampoline(self.fileno(), read=True, timeout=self.gettimeout(), timeout_exc=orig_socket.timeout) + trampoline(self.fileno(), read=True, timeout=self.gettimeout(), timeout_exc=timeout_exc('timed out')) return super(GreenSSLSocket, self).recvfrom_into(buffer, nbytes, flags) def unwrap(self): @@ -224,13 +235,13 @@ class GreenSSLSocket(__ssl.SSLSocket): except orig_socket.error, exc: if exc[0] in CONNECT_ERR: trampoline(self.fileno(), write=True, - timeout=end-time.time(), timeout_exc=orig_socket.timeout) + timeout=end-time.time(), timeout_exc=timeout_exc('timed out')) elif exc[0] in CONNECT_SUCCESS: return else: raise if time.time() >= end: - raise orig_socket.timeout + raise timeout_exc('timed out') def connect(self, addr): @@ -264,7 +275,7 @@ class GreenSSLSocket(__ssl.SSLSocket): if e[0] != errno.EWOULDBLOCK: raise trampoline(self.fileno(), read=True, timeout=self.gettimeout(), - timeout_exc=orig_socket.timeout) + timeout_exc=timeout_exc('timed out')) new_ssl = type(self)(newsock, keyfile=self.keyfile, @@ -276,7 +287,7 @@ class GreenSSLSocket(__ssl.SSLSocket): do_handshake_on_connect=self.do_handshake_on_connect, suppress_ragged_eofs=self.suppress_ragged_eofs) return (new_ssl, addr) - + SSLSocket = GreenSSLSocket diff --git a/eventlet/hubs/poll.py b/eventlet/hubs/poll.py index 0c72a85..c05a41c 100644 --- a/eventlet/hubs/poll.py +++ b/eventlet/hubs/poll.py @@ -7,7 +7,7 @@ import time from eventlet.hubs.hub import BaseHub, READ, WRITE EXC_MASK = select.POLLERR | select.POLLHUP -READ_MASK = select.POLLIN +READ_MASK = select.POLLIN | select.POLLPRI WRITE_MASK = select.POLLOUT class Hub(BaseHub): diff --git a/tests/stdlib/test_ssl.py b/tests/stdlib/test_ssl.py index faa4f31..b01d62b 100644 --- a/tests/stdlib/test_ssl.py +++ b/tests/stdlib/test_ssl.py @@ -30,9 +30,9 @@ patcher.inject('test.test_ssl', ('threading', threading), ('urllib', urllib)) -# these appear to not work due to some wonkiness in the threading -# module... skipping them for now (can't use SkipTest either because -# test_main doesn't understand it) +# these don't pass because nonblocking ssl sockets don't report +# when the socket is closed uncleanly, per the docstring on +# eventlet.green.GreenSSLSocket # *TODO: fix and restore these tests ThreadedTests.testProtocolSSL2 = lambda s: None ThreadedTests.testProtocolSSL3 = lambda s: None From 102e719428873acacac1cf750504f09ef1380d03 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 6 Dec 2009 23:52:12 -0800 Subject: [PATCH 09/14] Added a connect test. --- tests/ssl_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/ssl_test.py b/tests/ssl_test.py index 2506b4e..b2a2e27 100644 --- a/tests/ssl_test.py +++ b/tests/ssl_test.py @@ -40,6 +40,21 @@ class SSLTest(LimitedTestCase): greenio.shutdown_safe(client) client.close() server_coro.wait() + + def test_ssl_connect(self): + def serve(listener): + sock, addr = listener.accept() + stuff = sock.read(8192) + sock = api.ssl_listener(('127.0.0.1', 0), certificate_file, private_key_file) + server_coro = coros.execute(serve, sock) + + raw_client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ssl_client = util.wrap_ssl(raw_client) + ssl_client.connect(('127.0.0.1', sock.getsockname()[1])) + ssl_client.write('abc') + greenio.shutdown_safe(ssl_client) + ssl_client.close() + server_coro.wait() class SocketSSLTest(LimitedTestCase): From 173ab97db03155f1654afd691c915f0b37437d76 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 7 Dec 2009 13:28:45 -0800 Subject: [PATCH 10/14] Tpool tests now pass on Windows! --- AUTHORS | 4 ++-- eventlet/tpool.py | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/AUTHORS b/AUTHORS index 0ffb0c6..2d37d36 100644 --- a/AUTHORS +++ b/AUTHORS @@ -26,11 +26,11 @@ Thanks To * Chuck Thier, reporting a bug in processes.py * Brantley Harris, reporting bug #4 * Taso Du Val, reproing an exception squelching bug, saving children's lives ;-) -* R. Tyler Ballance, bug report on tpool on Windows, help with improving corolocal module, keen eye for redundancy, profile performance report +* R. Tyler Ballance, bug report on tpool on Windows, help with improving corolocal module, keen eye for redundancy, profile performance report, suggestion use flush that fixed tpool on Windows * Sergey Shepelev, PEP 8 police :-), reporting bug #5 * Luci Stanescu, for reporting twisted hub bug * Marcus Cavanaugh, for test case code that has been incredibly useful in tracking down bugs * Brian Brunswick, for many helpful questions and suggestions on the mailing list * Cesar Alaniz, for uncovering bugs of great import * the grugq, for contributing patches, suggestions, and use cases -* Ralf Schmitt, for wsgi/webob incompatibility bug report and suggested fix \ No newline at end of file +* Ralf Schmitt, for wsgi/webob incompatibility bug report and suggested fix diff --git a/eventlet/tpool.py b/eventlet/tpool.py index 9c55774..8c2c5b9 100644 --- a/eventlet/tpool.py +++ b/eventlet/tpool.py @@ -25,9 +25,9 @@ QUIET=False _rfile = _wfile = None def _signal_t2e(): - from eventlet import util - sent = util.__original_write__(_wfile.fileno(), ' ') - + _wfile.write(' ') + _wfile.flush() + _reqq = Queue(maxsize=-1) _rspq = Queue(maxsize=-1) @@ -36,9 +36,9 @@ def tpool_trampoline(): while(True): try: _c = _rfile.read(1) + assert(_c != "") except ValueError: break # will be raised when pipe is closed - assert(_c != "") while not _rspq.empty(): try: (e,rv) = _rspq.get(block=False) @@ -197,9 +197,7 @@ def setup(): sock.listen(50) csock = util.__original_socket__(socket.AF_INET, socket.SOCK_STREAM) csock.connect(('localhost', sock.getsockname()[1])) - csock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) nsock, addr = sock.accept() - nsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) _rfile = greenio.Green_fileobject(greenio.GreenSocket(csock)) _wfile = nsock.makefile() @@ -218,7 +216,8 @@ def killall(): _reqq.put(None) for thr in _threads.values(): thr.join() - api.kill(_coro) + if _coro: + api.kill(_coro) _rfile.close() _wfile.close() _setup_already = False From dd7257e2378ce7f41899d273bfbbb7910ac80448 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 7 Dec 2009 14:39:26 -0800 Subject: [PATCH 11/14] Refactored __init__ a little, skipped a processes-based tests on Windows. --- tests/__init__.py | 63 ++++++++++++++++++++++++++++------------- tests/processes_test.py | 20 +++++++++---- tests/saranwrap_test.py | 37 ++++++++++++++++++++++-- 3 files changed, 92 insertions(+), 28 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index d0b6e04..a1df6a8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -4,6 +4,9 @@ import os import errno import unittest +# convenience +main = unittest.main + def skipped(func): """ Decorator that marks a function as skipped. Uses nose's SkipTest exception if installed. Without nose, this will count skipped tests as passing tests.""" @@ -21,24 +24,39 @@ def skipped(func): return skipme -def skip_unless(requirement): - """ Decorator that skips a test if the *requirement* does not return True. - *requirement* can be a boolean or a callable that accepts one argument. - The callable will be called with the function to be decorated, and - should return True if the requirement is satisfied. +def skip_if(condition): + """ Decorator that skips a test if the *condition* evaluates True. + *condition* can be a boolean or a callable that accepts one argument. + The callable will be called with the function to be decorated, and + should return True to skip the test. """ - if isinstance(requirement, bool): - def skipped_wrapper(func): - if not requirement: - return skipped(func) - else: - return func - else: - def skipped_wrapper(func): - if not requirement(func): - return skipped(func) - else: - return func + def skipped_wrapper(func): + if isinstance(condition, bool): + result = condition + else: + result = condition(func) + if result: + return skipped(func) + else: + return func + return skipped_wrapper + + +def skip_unless(condition): + """ Decorator that skips a test if the *condition* does not return True. + *condition* can be a boolean or a callable that accepts one argument. + The callable will be called with the function to be decorated, and + should return True if the condition is satisfied. + """ + def skipped_wrapper(func): + if isinstance(condition, bool): + result = condition + else: + result = condition(func) + if not result: + return skipped(func) + else: + return func return skipped_wrapper @@ -55,10 +73,15 @@ def requires_twisted(func): def skip_with_libevent(func): """ Decorator that skips a test if we're using the libevent hub.""" - def requirement(_f): + def using_libevent(_f): from eventlet.api import get_hub - return not('libevent' in type(get_hub()).__module__) - return skip_unless(requirement)(func) + return 'libevent' in type(get_hub()).__module__ + return skip_if(using_libevent)(func) + + +def skip_on_windows(func): + import sys + return skip_if(sys.platform.startswith('win'))(func) class TestIsTakingTooLong(Exception): diff --git a/tests/processes_test.py b/tests/processes_test.py index 2bcd81e..17355ed 100644 --- a/tests/processes_test.py +++ b/tests/processes_test.py @@ -1,12 +1,14 @@ import sys -from unittest import TestCase, main +from tests import LimitedTestCase, main, skip_on_windows from eventlet import processes, api -class TestEchoPool(TestCase): +class TestEchoPool(LimitedTestCase): def setUp(self): + super(TestEchoPool, self).setUp() self.pool = processes.ProcessPool('echo', ["hello"]) + @skip_on_windows def test_echo(self): result = None @@ -17,6 +19,7 @@ class TestEchoPool(TestCase): self.pool.put(proc) self.assertEquals(result, 'hello\n') + @skip_on_windows def test_read_eof(self): proc = self.pool.get() try: @@ -24,18 +27,21 @@ class TestEchoPool(TestCase): self.assertRaises(processes.DeadProcess, proc.read) finally: self.pool.put(proc) - + + @skip_on_windows def test_empty_echo(self): p = processes.Process('echo', ['-n']) self.assertEquals('', p.read()) self.assertRaises(processes.DeadProcess, p.read) -class TestCatPool(TestCase): +class TestCatPool(LimitedTestCase): def setUp(self): + super(TestCatPool, self).setUp() api.sleep(0) self.pool = processes.ProcessPool('cat') + @skip_on_windows def test_cat(self): result = None @@ -49,6 +55,7 @@ class TestCatPool(TestCase): self.assertEquals(result, 'goodbye') + @skip_on_windows def test_write_to_dead(self): result = None @@ -61,6 +68,7 @@ class TestCatPool(TestCase): finally: self.pool.put(proc) + @skip_on_windows def test_close(self): result = None @@ -73,10 +81,12 @@ class TestCatPool(TestCase): self.pool.put(proc) -class TestDyingProcessesLeavePool(TestCase): +class TestDyingProcessesLeavePool(LimitedTestCase): def setUp(self): + super(TestDyingProcessesLeavePool, self).setUp() self.pool = processes.ProcessPool('echo', ['hello'], max_size=1) + @skip_on_windows def test_dead_process_not_inserted_into_pool(self): proc = self.pool.get() try: diff --git a/tests/saranwrap_test.py b/tests/saranwrap_test.py index 7d6c580..03fb1e2 100644 --- a/tests/saranwrap_test.py +++ b/tests/saranwrap_test.py @@ -5,7 +5,7 @@ import os import sys import tempfile import time -import unittest +from tests import LimitedTestCase, main, skip_on_windows import re import StringIO @@ -31,12 +31,13 @@ class CoroutineCallingClass(object): return self._my_dict -class TestSaranwrap(unittest.TestCase): +class TestSaranwrap(LimitedTestCase): def assert_server_exists(self, prox): self.assert_(saranwrap.status(prox)) prox.foo = 0 self.assertEqual(0, prox.foo) + @skip_on_windows def test_wrap_tuple(self): my_tuple = (1, 2) prox = saranwrap.wrap(my_tuple) @@ -44,6 +45,7 @@ class TestSaranwrap(unittest.TestCase): self.assertEqual(prox[1], 2) self.assertEqual(len(my_tuple), 2) + @skip_on_windows def test_wrap_string(self): my_object = "whatever" prox = saranwrap.wrap(my_object) @@ -51,6 +53,7 @@ class TestSaranwrap(unittest.TestCase): self.assertEqual(len(my_object), len(prox)) self.assertEqual(my_object.join(['a', 'b']), prox.join(['a', 'b'])) + @skip_on_windows def test_wrap_uniterable(self): # here we're treating the exception as just a normal class prox = saranwrap.wrap(FloatingPointError()) @@ -62,6 +65,7 @@ class TestSaranwrap(unittest.TestCase): self.assertRaises(IndexError, index) self.assertRaises(TypeError, key) + @skip_on_windows def test_wrap_dict(self): my_object = {'a':1} prox = saranwrap.wrap(my_object) @@ -71,6 +75,7 @@ class TestSaranwrap(unittest.TestCase): self.assertEqual('saran:' + repr(my_object), repr(prox)) self.assertEqual('saran:' + `my_object`, `prox`) + @skip_on_windows def test_wrap_module_class(self): prox = saranwrap.wrap(re) self.assertEqual(saranwrap.Proxy, type(prox)) @@ -78,6 +83,7 @@ class TestSaranwrap(unittest.TestCase): self.assertEqual(exp.flags, 0) self.assert_(repr(prox.compile)) + @skip_on_windows def test_wrap_eq(self): prox = saranwrap.wrap(re) exp1 = prox.compile('.') @@ -86,6 +92,7 @@ class TestSaranwrap(unittest.TestCase): exp3 = prox.compile('/') self.assert_(exp1 != exp3) + @skip_on_windows def test_wrap_nonzero(self): prox = saranwrap.wrap(re) exp1 = prox.compile('.') @@ -93,6 +100,7 @@ class TestSaranwrap(unittest.TestCase): prox2 = saranwrap.Proxy([1, 2, 3]) self.assert_(bool(prox2)) + @skip_on_windows def test_multiple_wraps(self): prox1 = saranwrap.wrap(re) prox2 = saranwrap.wrap(re) @@ -101,6 +109,7 @@ class TestSaranwrap(unittest.TestCase): del x2 x3 = prox2.compile('.') + @skip_on_windows def test_dict_passthru(self): prox = saranwrap.wrap(StringIO) x = prox.StringIO('a') @@ -108,25 +117,30 @@ class TestSaranwrap(unittest.TestCase): # try it all on one line just for the sake of it self.assertEqual(type(saranwrap.wrap(StringIO).StringIO('a').__dict__), saranwrap.ObjectProxy) + @skip_on_windows def test_is_value(self): server = saranwrap.Server(None, None, None) self.assert_(server.is_value(None)) + @skip_on_windows def test_wrap_getitem(self): prox = saranwrap.wrap([0,1,2]) self.assertEqual(prox[0], 0) + @skip_on_windows def test_wrap_setitem(self): prox = saranwrap.wrap([0,1,2]) prox[1] = 2 self.assertEqual(prox[1], 2) + @skip_on_windows def test_raising_exceptions(self): prox = saranwrap.wrap(re) def nofunc(): prox.never_name_a_function_like_this() self.assertRaises(AttributeError, nofunc) + @skip_on_windows def test_unpicklable_server_exception(self): prox = saranwrap.wrap(saranwrap) def unpickle(): @@ -137,6 +151,7 @@ class TestSaranwrap(unittest.TestCase): # It's basically dead #self.assert_server_exists(prox) + @skip_on_windows def test_pickleable_server_exception(self): prox = saranwrap.wrap(saranwrap) def fperror(): @@ -145,11 +160,13 @@ class TestSaranwrap(unittest.TestCase): self.assertRaises(FloatingPointError, fperror) self.assert_server_exists(prox) + @skip_on_windows def test_print_does_not_break_wrapper(self): prox = saranwrap.wrap(saranwrap) prox.print_string('hello') self.assert_server_exists(prox) + @skip_on_windows def test_stderr_does_not_break_wrapper(self): prox = saranwrap.wrap(saranwrap) prox.err_string('goodbye') @@ -158,6 +175,7 @@ class TestSaranwrap(unittest.TestCase): def assertLessThan(self, a, b): self.assert_(a < b, "%s is not less than %s" % (a, b)) + @skip_on_windows def test_status(self): prox = saranwrap.wrap(time) a = prox.gmtime(0) @@ -176,6 +194,7 @@ class TestSaranwrap(unittest.TestCase): prox2 = saranwrap.wrap(re) self.assert_(status['pid'] != saranwrap.status(prox2)['pid']) + @skip_on_windows def test_del(self): prox = saranwrap.wrap(time) delme = prox.gmtime(0) @@ -189,11 +208,13 @@ class TestSaranwrap(unittest.TestCase): #print status_after['objects'] self.assertLessThan(status_after['object_count'], status_before['object_count']) + @skip_on_windows def test_contains(self): prox = saranwrap.wrap({'a':'b'}) self.assert_('a' in prox) self.assert_('x' not in prox) + @skip_on_windows def test_variable_and_keyword_arguments_with_function_calls(self): import optparse prox = saranwrap.wrap(optparse) @@ -202,6 +223,7 @@ class TestSaranwrap(unittest.TestCase): opts,args = parser.parse_args(["-nfoo"]) self.assertEqual(opts.n, 'foo') + @skip_on_windows def test_original_proxy_going_out_of_scope(self): def make_re(): prox = saranwrap.wrap(re) @@ -224,6 +246,7 @@ class TestSaranwrap(unittest.TestCase): except AttributeError, e: pass + @skip_on_windows def test_not_inheriting_pythonpath(self): # construct a fake module in the temp directory temp_dir = tempfile.mkdtemp("saranwrap_test") @@ -253,6 +276,7 @@ sys_path = sys.path""") shutil.rmtree(temp_dir) sys.path.remove(temp_dir) + @skip_on_windows def test_contention(self): from tests import saranwrap_test prox = saranwrap.wrap(saranwrap_test) @@ -265,6 +289,7 @@ sys_path = sys.path""") for waiter in waiters: waiter.wait() + @skip_on_windows def test_copy(self): import copy compound_object = {'a':[1,2,3]} @@ -278,12 +303,14 @@ sys_path = sys.path""") make_assertions(copy.copy(prox)) make_assertions(copy.deepcopy(prox)) + @skip_on_windows def test_list_of_functions(self): return # this test is known to fail, we can implement it sometime in the future if we wish from tests import saranwrap_test prox = saranwrap.wrap([saranwrap_test.list_maker]) self.assertEquals(list_maker(), prox[0]()) + @skip_on_windows def test_under_the_hood_coroutines(self): # so, we want to write a class which uses a coroutine to call # a function. Then we want to saranwrap that class, have @@ -302,6 +329,7 @@ sys_path = sys.path""") 'random' in obj_proxy.get_dict(), 'Coroutine in saranwrapped object did not run') + @skip_on_windows def test_child_process_death(self): prox = saranwrap.wrap({}) pid = saranwrap.getpid(prox) @@ -310,17 +338,20 @@ sys_path = sys.path""") api.sleep(0.1) # need to let the signal handler run self.assertRaises(OSError, os.kill, pid, 0) # raises OSError if pid doesn't exist + @skip_on_windows def test_detection_of_server_crash(self): # make the server crash here pass + @skip_on_windows def test_equality_with_local_object(self): # we'll implement this if there's a use case for it pass + @skip_on_windows def test_non_blocking(self): # here we test whether it's nonblocking pass if __name__ == '__main__': - unittest.main() + main() From a1aca0110f6c3a444395302b5571f2f38407e8fd Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 7 Dec 2009 18:36:42 -0800 Subject: [PATCH 12/14] Buncha tiny windows-specific fixes. --- eventlet/wsgi.py | 2 +- tests/test__event.py | 1 + tests/test__socket_errors.py | 3 ++- tests/wsgi_test.py | 14 +++++++++++--- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/eventlet/wsgi.py b/eventlet/wsgi.py index d4e7e69..fc0eacd 100644 --- a/eventlet/wsgi.py +++ b/eventlet/wsgi.py @@ -154,7 +154,7 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler): except greenio.SSL.ZeroReturnError: self.raw_requestline = '' except socket.error, e: - if e[0] != errno.EBADF: + if e[0] != errno.EBADF and e[0] != 10053: raise self.raw_requestline = '' diff --git a/tests/test__event.py b/tests/test__event.py index 2a4e418..d6f2ebd 100644 --- a/tests/test__event.py +++ b/tests/test__event.py @@ -22,6 +22,7 @@ class TestEvent(LimitedTestCase): obj = Exception() e.send(exc=obj) sleep(0) + sleep(0) assert log == [('catched', obj)], log def test_send(self): diff --git a/tests/test__socket_errors.py b/tests/test__socket_errors.py index 5d3aee7..502acec 100644 --- a/tests/test__socket_errors.py +++ b/tests/test__socket_errors.py @@ -12,9 +12,10 @@ class TestSocketErrors(unittest.TestCase): s = socket.socket() try: s.connect(('127.0.0.1', 81)) + self.fail("Shouldn't have connected") except socket.error, ex: code, text = ex.args - assert code in [111, 61], (code, text) + assert code in [111, 61, 10061], (code, text) assert 'refused' in text.lower(), (code, text) if __name__=='__main__': diff --git a/tests/wsgi_test.py b/tests/wsgi_test.py index 01567d0..131072e 100644 --- a/tests/wsgi_test.py +++ b/tests/wsgi_test.py @@ -84,7 +84,12 @@ class ConnectionClosed(Exception): def read_http(sock): fd = sock.makeGreenFile() - response_line = fd.readline() + try: + response_line = fd.readline() + except socket.error, exc: + if exc[0] == 10053: + raise ConnectionClosed + raise if not response_line: raise ConnectionClosed raw_headers = fd.readuntil('\r\n\r\n').strip() @@ -189,8 +194,11 @@ class TestHttpd(LimitedTestCase): fd = sock.makeGreenFile() fd.write(request) result = fd.readline() - status = result.split(' ')[1] - self.assertEqual(status, '414') + if result: + # windows closes the socket before the data is flushed, + # so we never get anything back + status = result.split(' ')[1] + self.assertEqual(status, '414') fd.close() def test_007_get_arg(self): From 77eeb4fbf5ae0c8fd9a16b2c511ee6b2533eba7c Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Fri, 11 Dec 2009 11:39:57 -0800 Subject: [PATCH 13/14] Sometimes the saranwrap tests were timing out on loaded hosts simply due to the fact that process launching is time-consuming. Tripled the timeout and things seem better. --- tests/saranwrap_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/saranwrap_test.py b/tests/saranwrap_test.py index 03fb1e2..ce84a45 100644 --- a/tests/saranwrap_test.py +++ b/tests/saranwrap_test.py @@ -32,6 +32,7 @@ class CoroutineCallingClass(object): class TestSaranwrap(LimitedTestCase): + TEST_TIMEOUT=3 def assert_server_exists(self, prox): self.assert_(saranwrap.status(prox)) prox.foo = 0 From a9d834b2dd08c83fc61ce53b8fdd22f1b10f8f42 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Fri, 11 Dec 2009 11:46:50 -0800 Subject: [PATCH 14/14] Something started listening on port 81, and this test started failing. Changed so that it selects a port in a more foolproof manner. --- tests/test__socket_errors.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test__socket_errors.py b/tests/test__socket_errors.py index 502acec..573568b 100644 --- a/tests/test__socket_errors.py +++ b/tests/test__socket_errors.py @@ -9,9 +9,16 @@ else: class TestSocketErrors(unittest.TestCase): def test_connection_refused(self): + # open and close a dummy server to find an unused port + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.bind(('127.0.0.1', 0)) + server.listen(1) + port = server.getsockname()[1] + server.close() + del server s = socket.socket() try: - s.connect(('127.0.0.1', 81)) + s.connect(('127.0.0.1', port)) self.fail("Shouldn't have connected") except socket.error, ex: code, text = ex.args