From 30f3265012518cd49c94bee8869f6708aa63073d Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Fri, 2 Sep 2011 15:51:04 -0400 Subject: [PATCH 01/41] first stab at supporting multiple senders and receivers. incomplete. --- eventlet/green/zmq.py | 141 +++++++++++++++++++++++++++++++++++++++--- tests/zmq_test.py | 57 +++++++++++++++++ 2 files changed, 190 insertions(+), 8 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 187e0c4..0c33c6b 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -1,13 +1,15 @@ """The :mod:`zmq` module wraps the :class:`Socket` and :class:`Context` found in :mod:`pyzmq ` to be non blocking """ __zmq__ = __import__('zmq') -from eventlet import sleep +from eventlet import sleep, hubs from eventlet.hubs import trampoline, _threadlocal from eventlet.patcher import slurp_properties +from eventlet.support import greenlets as greenlet __patched__ = ['Context', 'Socket'] slurp_properties(__zmq__, globals(), ignore=__patched__) +from collections import deque def Context(io_threads=1): """Factory function replacement for :class:`zmq.core.context.Context` @@ -42,21 +44,44 @@ class _Context(__zmq__.Context): """ return Socket(self, socket_type) + +# see http://api.zeromq.org/2-1:zmq-socket for explanation of socket types +_multi_reader_types = set([__zmq__.XREP, __zmq__.XREQ, __zmq__.SUB, __zmq__.PULL, __zmq__.PAIR]) +_multi_writer_types = set([__zmq__.XREP, __zmq__.XREQ, __zmq__.PUB, __zmq__.PUSH, __zmq__.PAIR]) + class Socket(__zmq__.Socket): """Green version of :class:`zmq.core.socket.Socket - The following four methods are overridden: - - * _send_message - * _send_copy - * _recv_message - * _recv_copy - + The following two methods are always overridden: + * send + * recv To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or recieving is deferred to the hub (using :func:`eventlet.hubs.trampoline`) if a ``zmq.EAGAIN`` (retry) error is raised + + For some socket types, where multiple greenthreads could be + calling send or recv at the same time, these methods are also + overridden: + * send_multipart + * recv_multipart + """ + def __init__(self, *args, **kwargs): + super(Socket, self).__init__(*args, **kwargs) + + if False and self.socket_type in _multi_writer_types: + # support multiple greenthreads writing at the same time + self._writers = deque() + self.send = self._xsafe_send + self.send_multipart = self._xsafe_send_multipart + + if False and self.socket_type in _multi_reader_types: + # support multiple greenthreads reading at the same time + self._readers = deque() + self.recv = self._xsafe_recv + self.recv_multipart = self._xsafe_recv_multipart + def _sock_wait(self, read=False, write=False): """ First checks if there are events in the socket, to avoid @@ -118,4 +143,104 @@ class Socket(__zmq__.Socket): if e.errno != EAGAIN: raise + def _xsafe_send(self, msg, flags=0, copy=True, track=False): + """ + A send method that's safe to use when multiple greenthreads + are calling send, send_multipart, recv and recv_multipart on + the same socket. + """ + if flags & __zmq__.NOBLOCK: + super(Socket, self).send(msg, flags=flags, track=track, copy=copy) + return + flags |= __zmq__.NOBLOCK + + if self._writers: + self._writers.append((msg, flags, copy, track, greenlet.getcurrent())) + if hubs.get_hub().switch(): + # msg was sent by another greenthread + return + else: + pass + else: + self._writers.append((msg, flags, copy, track, greenlet.getcurrent())) + + while True: + try: + if (self.getsockopt(__zmq__.EVENTS) & __zmq__.POLLOUT): + super(Socket, self).send(msg, flags=flags, track=track, + copy=copy) + + + + self._sock_wait(write=True) + super(Socket, self).send(msg, flags=flags, track=track, + copy=copy) + return + except __zmq__.ZMQError, e: + if e.errno != EAGAIN: + raise + + def _xsafe_send_multipart(self, msg_parts, flags=0, copy=True, track=False): + """ + A send_multipart method that's safe to use when multiple + greenthreads are calling send, send_multipart, recv and + recv_multipart on the same socket. + + Ensure multipart messages are not interleaved. + """ + + self._writers.append((list(reversed(msg)), flags, copy, track, greenlet.getcurrent())) + if len(self._writers) == 1: + # no blocked writers + + pass + + + def _send_queued(self, ): + """ + Send as many msgs from the writers deque as possible. Wake up + the greenthreads for messages that are sent. + """ + writers = self.writers + hub = hubs.get_hub() + + while writers: + msg, flags, copy, track, writer = writers[0] + + if isinstance(msg, list): + is_list = True + m = msg[-1] + else: + is_list = False + m = msg + try: + super(Socket, self).send(m, flags=flags, track=track, + copy=copy) + hub.schedule_call_global(0, writer.switch, True) + except (SystemExit, KeyboardInterrupt): + raise + except __zmq__.ZMQError, e: + if e.errno == EAGAIN: + + pass + else: + hub.schedule_call_global(0, writer.throw, e) + + + def _xsafe_recv(self, flags=0, copy=True, track=False): + """ + A recv method that's safe to use when multiple greenthreads + are calling send, send_multipart, recv and recv_multipart on + the same socket. + """ + pass + + + def _xsafe_recv_multipart(self, flags=0, copy=True, track=False): + """ + A recv method that's safe to use when multiple greenthreads + are calling send, send_multipart, recv and recv_multipart on + the same socket. + """ + pass diff --git a/tests/zmq_test.py b/tests/zmq_test.py index 26d0bda..f09e872 100644 --- a/tests/zmq_test.py +++ b/tests/zmq_test.py @@ -238,6 +238,63 @@ got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) self.assertRaisesErrno(zmq.EAGAIN, rep.recv, zmq.NOBLOCK) self.assertRaisesErrno(zmq.EAGAIN, rep.recv, zmq.NOBLOCK, True) + @skip_unless(zmq_supported) + def test_send_during_recv(self): + sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) + sleep() + done = event.Event() + + def slow_rx(): + self.assertEqual(sender.recv(), "done") + done.send(0) + + def tx(): + tx_i = 0 + while tx_i <= 1000: + sender.send(str(tx_i)) + tx_i += 1 + + def rx(): + while True: + rx_i = receiver.recv() + if rx_i == "1000": + receiver.send('done') + return + spawn(slow_rx) + spawn(tx) + spawn(rx) + final_i = done.wait() + self.assertEqual(final_i, 0) + + # Need someway to ensure a thread is blocked on send. This method + # below uses too much memory. Try adjust watermarks or other + # socket opts? + + # @skip_unless(zmq_supported) + # def test_recv_during_send(self): + # sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) + # sleep() + # done = event.Event() + + # def tx(): + # msg = "0" * 1024 + # while True: + # sender.send(msg) + + # def rx(): + # self.assertEqual(sender.recv(), "done") + # sender_thread.kill() + # done.send(0) + + # def single_tx(): + # receiver.send("done") + + # sender_thread = spawn(tx) + # sleep() + # spawn(rx) + # spawn(single_tx) + # final_i = done.wait() + # self.assertEqual(final_i, 0) class TestThreadedContextAccess(TestCase): From e4d0e91f793be921060317a3c2601aac99e6f45c Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Sat, 3 Sep 2011 09:07:30 -0400 Subject: [PATCH 02/41] send/send_multipart from multiple greenthreads working, maybe... --- eventlet/green/zmq.py | 217 ++++++++++++++++++++++++++++++------------ 1 file changed, 154 insertions(+), 63 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 0c33c6b..647f4c6 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -49,6 +49,15 @@ class _Context(__zmq__.Context): _multi_reader_types = set([__zmq__.XREP, __zmq__.XREQ, __zmq__.SUB, __zmq__.PULL, __zmq__.PAIR]) _multi_writer_types = set([__zmq__.XREP, __zmq__.XREQ, __zmq__.PUB, __zmq__.PUSH, __zmq__.PAIR]) +_disable_send_types = set([__zmq__.SUB, __zmq__.PULL]) +_disable_recv_types = set([__zmq__.PUB, __zmq__.PUSH]) + + +# TODO: +# - Ensure that recv* and send* methods raise error when called on a +# closed socket. They should not block. +# - Return correct message tracker from send* methods + class Socket(__zmq__.Socket): """Green version of :class:`zmq.core.socket.Socket @@ -67,20 +76,25 @@ class Socket(__zmq__.Socket): """ - def __init__(self, *args, **kwargs): - super(Socket, self).__init__(*args, **kwargs) + def __init__(self, context, socket_type): + super(Socket, self).__init__(context, socket_type) - if False and self.socket_type in _multi_writer_types: + # customize send and recv functions based on socket type + if socket_type in _multi_writer_types: # support multiple greenthreads writing at the same time self._writers = deque() self.send = self._xsafe_send self.send_multipart = self._xsafe_send_multipart + elif socket_type in _disable_send_types: + self.send = self.send_multipart = _send_not_supported - if False and self.socket_type in _multi_reader_types: + if socket_type in _multi_reader_types: # support multiple greenthreads reading at the same time self._readers = deque() self.recv = self._xsafe_recv self.recv_multipart = self._xsafe_recv_multipart + elif socket_type in _disable_recv_types: + self.recv = self.recv_multipart = _recv_not_supported def _sock_wait(self, read=False, write=False): """ @@ -107,7 +121,7 @@ class Socket(__zmq__.Socket): called in real code. """ if flags & __zmq__.NOBLOCK: - super(Socket, self).send(msg, flags=flags, track=track, copy=copy) + super(Socket, self).send(msg, flags=flags, copy=copy, track=track) return flags |= __zmq__.NOBLOCK @@ -115,9 +129,7 @@ class Socket(__zmq__.Socket): while True: try: self._sock_wait(write=True) - super(Socket, self).send(msg, flags=flags, track=track, - copy=copy) - return + return super(Socket, self).send(msg, flags=flags, copy=copy, track=track) except __zmq__.ZMQError, e: if e.errno != EAGAIN: raise @@ -129,20 +141,44 @@ class Socket(__zmq__.Socket): called in real code. """ if flags & __zmq__.NOBLOCK: - return super(Socket, self).recv(flags=flags, track=track, copy=copy) + return super(Socket, self).recv(flags=flags, copy=copy, track=track) flags |= __zmq__.NOBLOCK while True: try: self._sock_wait(read=True) - m = super(Socket, self).recv(flags=flags, track=track, copy=copy) - if m is not None: - return m + return super(Socket, self).recv(flags=flags, copy=copy, track=track) except __zmq__.ZMQError, e: if e.errno != EAGAIN: raise + def _send_not_supported(self, msg, flags=0, copy=True, track=False): + raise __zmq__.ZMQError(__zmq__.ENOTSUP) + + def _recv_not_supported(self, flags=0, copy=True, track=False): + raise __zmq__.ZMQError(__zmq__.ENOTSUP) + + + def _xsafe_sock_wait(self, read=False, write=False): + """ + First checks if there are events in the socket, to avoid + edge trigger problems with race conditions. Then if there + are none it will trampoline and when coming back check + for the events. + """ + events = self.getsockopt(__zmq__.EVENTS) + + if read and (events & __zmq__.POLLIN): + return events + elif write and (events & __zmq__.POLLOUT): + return events + else: + # ONLY trampoline on read events for the zmq FD + trampoline(self.getsockopt(__zmq__.FD), read=True) + return self.getsockopt(__zmq__.EVENTS) + + def _xsafe_send(self, msg, flags=0, copy=True, track=False): """ A send method that's safe to use when multiple greenthreads @@ -150,37 +186,10 @@ class Socket(__zmq__.Socket): the same socket. """ if flags & __zmq__.NOBLOCK: - super(Socket, self).send(msg, flags=flags, track=track, copy=copy) - return - - flags |= __zmq__.NOBLOCK - - if self._writers: - self._writers.append((msg, flags, copy, track, greenlet.getcurrent())) - if hubs.get_hub().switch(): - # msg was sent by another greenthread - return - else: - pass - else: - self._writers.append((msg, flags, copy, track, greenlet.getcurrent())) - - while True: - try: - if (self.getsockopt(__zmq__.EVENTS) & __zmq__.POLLOUT): - super(Socket, self).send(msg, flags=flags, track=track, - copy=copy) - - - - self._sock_wait(write=True) - super(Socket, self).send(msg, flags=flags, track=track, - copy=copy) - return - except __zmq__.ZMQError, e: - if e.errno != EAGAIN: - raise + return super(Socket, self).send(msg, flags=flags, copy=copy, track=track) + self._xsafe_inner_send(msg, flags, copy, track) + def _xsafe_send_multipart(self, msg_parts, flags=0, copy=True, track=False): """ A send_multipart method that's safe to use when multiple @@ -190,52 +199,133 @@ class Socket(__zmq__.Socket): Ensure multipart messages are not interleaved. """ - self._writers.append((list(reversed(msg)), flags, copy, track, greenlet.getcurrent())) - if len(self._writers) == 1: - # no blocked writers + if flags & __zmq__.NOBLOCK: + return super(Socket, self).send_multipart(msg_parts, flags=flags, copy=copy, track=track) + + self._xsafe_inner_send(list(msg_parts), flags, copy, track) + + def _xsafe_inner_send(self, msg, flags, copy, track): + is_listening = bool(self._writers or self._readers) + + self._writers.append((msg, flags | __zmq__.NOBLOCK, copy, track, greenlet.getcurrent())) + if is_listening: + # Other readers or writers are blocked. If this the first writer, it may be possible to send immediately + if len(self._writers) == 1: + result = self._send_queued() + if not self._writers: + # success! + return result - pass + # other readers or writers are blocked so this greenthread must wait its turn + result = hubs.get_hub().switch() + if result is False: + # msg was not yet sent, but this thread was woken up + # so that it could process the queues + return self._process_queues() + else: + # msg was sent by another greenthread + return result + else: + return self._process_queues() + def _process_queues(self): + """ If there are readers or writers queued, this method tries + to recv or send messages and ensures processing continues + either in this greenthread or in another one. """ + readers = self._readers + writers = self._writers - def _send_queued(self, ): + send_result = None + recv_result = None + while True: + events = self.getsockopt(__zmq__.EVENTS) + try: + if readers and (events & __zmq__.POLLIN): + recv_result = self._recv_queued() + + if writers and (events & __zmq__.POLLOUT): + send_result = self._send_queued() + except (SystemExit, KeyboardInterrupt): + raise + except: + # an error occurred for this greenthread's send/recv + # call. Wake another thread to continue processing. + if readers: + hubs.get_hub().schedule_call_global(0, readers[0].switch, False) + elif writers: + hubs.get_hub().schedule_call_global(0, writers[0][-1].switch, False) + raise + + # send and recv cannot continue right now. If there are + # more readers or writers queued, either trampoline or + # wake another greenthread. + current = greelnet.getcurrent() + if (readers and readers[0] is current) or (writers and writers[0][-1] is current): + # Only trampoline if this thread is the next reader or writer, + # and ONLY trampoline on read events for zmq FDs. + trampoline(self.getsockopt(__zmq__.FD), read=True) + else: + if readers: + hubs.get_hub().schedule_call_global(0, readers[0].switch, False) + elif writers: + hubs.get_hub().schedule_call_global(0, writers[0][-1].switch, False) + return send_result or recv_result + + + def _send_queued(self): """ Send as many msgs from the writers deque as possible. Wake up the greenthreads for messages that are sent. """ writers = self.writers + current = greenlet.getcurrent() hub = hubs.get_hub() + super_send = super(Socket, self).send + super_send_multipart = super(Socket, self).send_multipart + + result = None while writers: msg, flags, copy, track, writer = writers[0] - - if isinstance(msg, list): - is_list = True - m = msg[-1] - else: - is_list = False - m = msg try: - super(Socket, self).send(m, flags=flags, track=track, - copy=copy) - hub.schedule_call_global(0, writer.switch, True) + if isinstance(msg, list): + r = super_send_multipart(msg, flags=flags, copy=copy, track=track) + else: + r = super_send(msg, flags=flags, copy=copy, track=track) + + # remember this thread's result + if current is writer: + result = r except (SystemExit, KeyboardInterrupt): raise except __zmq__.ZMQError, e: if e.errno == EAGAIN: - - pass + return result else: - hub.schedule_call_global(0, writer.throw, e) - + # message failed to send + writers.popleft() + if current is writer: + raise + else: + hub.schedule_call_global(0, writer.throw, e) + continue + # move to the next msg + writers.popleft() + # wake writer + if current is not writer: + hub.schedule_call_global(0, writer.switch, r) + return result + def _xsafe_recv(self, flags=0, copy=True, track=False): """ A recv method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. """ - pass + if flags & __zmq__.NOBLOCK: + return super(Socket, self).recv(flags=flags, copy=copy, track=track) def _xsafe_recv_multipart(self, flags=0, copy=True, track=False): """ @@ -243,4 +333,5 @@ class Socket(__zmq__.Socket): are calling send, send_multipart, recv and recv_multipart on the same socket. """ - pass + if flags & __zmq__.NOBLOCK: + return super(Socket, self).recv_multipart(flags=flags, copy=copy, track=track) From 77157811d47acbd2307dba6549b507484a5e7178 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Sat, 3 Sep 2011 09:31:18 -0400 Subject: [PATCH 03/41] implement recv for multiple greenthreads --- eventlet/green/zmq.py | 140 +++++++++++++++++++++++++++++++++--------- 1 file changed, 112 insertions(+), 28 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 647f4c6..7e92a4f 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -57,6 +57,7 @@ _disable_recv_types = set([__zmq__.PUB, __zmq__.PUSH]) # - Ensure that recv* and send* methods raise error when called on a # closed socket. They should not block. # - Return correct message tracker from send* methods +# - Make MessageTracker.wait zmq friendly class Socket(__zmq__.Socket): """Green version of :class:`zmq.core.socket.Socket @@ -79,6 +80,8 @@ class Socket(__zmq__.Socket): def __init__(self, context, socket_type): super(Socket, self).__init__(context, socket_type) + self._writers = None + self._readers = None # customize send and recv functions based on socket type if socket_type in _multi_writer_types: # support multiple greenthreads writing at the same time @@ -86,7 +89,7 @@ class Socket(__zmq__.Socket): self.send = self._xsafe_send self.send_multipart = self._xsafe_send_multipart elif socket_type in _disable_send_types: - self.send = self.send_multipart = _send_not_supported + self.send = self.send_multipart = self._send_not_supported if socket_type in _multi_reader_types: # support multiple greenthreads reading at the same time @@ -94,7 +97,7 @@ class Socket(__zmq__.Socket): self.recv = self._xsafe_recv self.recv_multipart = self._xsafe_recv_multipart elif socket_type in _disable_recv_types: - self.recv = self.recv_multipart = _recv_not_supported + self.recv = self.recv_multipart = self._recv_not_supported def _sock_wait(self, read=False, write=False): """ @@ -207,10 +210,12 @@ class Socket(__zmq__.Socket): def _xsafe_inner_send(self, msg, flags, copy, track): is_listening = bool(self._writers or self._readers) - self._writers.append((msg, flags | __zmq__.NOBLOCK, copy, track, greenlet.getcurrent())) + self._writers.append((greenlet.getcurrent(), msg, flags | __zmq__.NOBLOCK, copy, track)) if is_listening: - # Other readers or writers are blocked. If this the first writer, it may be possible to send immediately + # Other readers or writers are blocked. If this is the + # first writer, it may be possible to send immediately if len(self._writers) == 1: + # TODO: Check EVENTS first? result = self._send_queued() if not self._writers: # success! @@ -228,6 +233,57 @@ class Socket(__zmq__.Socket): else: return self._process_queues() + def _xsafe_recv(self, flags=0, copy=True, track=False): + """ + A recv method that's safe to use when multiple greenthreads + are calling send, send_multipart, recv and recv_multipart on + the same socket. + """ + + if flags & __zmq__.NOBLOCK: + return super(Socket, self).recv(flags=flags, copy=copy, track=track) + + return self._xsafe_inner_recv(False, flags, copy, track) + + def _xsafe_recv_multipart(self, flags=0, copy=True, track=False): + """ + A recv method that's safe to use when multiple greenthreads + are calling send, send_multipart, recv and recv_multipart on + the same socket. + """ + if flags & __zmq__.NOBLOCK: + return super(Socket, self).recv_multipart(flags=flags, copy=copy, track=track) + + return self._xsafe_inner_recv(True, flags, copy, track) + + def _xsafe_inner_recv(self, multi, flags, copy, track): + is_listening = bool(self._writers or self._readers) + + self._readers.append((greenlet.getcurrent(), multi, flags | __zmq__.NOBLOCK, copy, track)) + + if is_listening: + # Other readers or writers are blocked. If this is the + # first reader, it may be possible to recv immediately + if len(self._readers) == 1: + # TODO: Check EVENTS first? + result = self._recv_queued() + if not self._readers: + # success! + return result + + # other readers or writers are blocked so this greenthread must wait its turn + result = hubs.get_hub().switch() + if result is False: + # msg was not yet received, but this thread was woken up + # so that it could process the queues + return self._process_queues() + else: + # msg was received + return result + else: + return self._process_queues() + + def _process_queues(self): """ If there are readers or writers queued, this method tries to recv or send messages and ensures processing continues @@ -251,24 +307,24 @@ class Socket(__zmq__.Socket): # an error occurred for this greenthread's send/recv # call. Wake another thread to continue processing. if readers: - hubs.get_hub().schedule_call_global(0, readers[0].switch, False) + hubs.get_hub().schedule_call_global(0, readers[0][0].switch, False) elif writers: - hubs.get_hub().schedule_call_global(0, writers[0][-1].switch, False) + hubs.get_hub().schedule_call_global(0, writers[0][0].switch, False) raise # send and recv cannot continue right now. If there are # more readers or writers queued, either trampoline or # wake another greenthread. - current = greelnet.getcurrent() - if (readers and readers[0] is current) or (writers and writers[0][-1] is current): + current = greenlet.getcurrent() + if (readers and readers[0][0] is current) or (writers and writers[0][0] is current): # Only trampoline if this thread is the next reader or writer, # and ONLY trampoline on read events for zmq FDs. trampoline(self.getsockopt(__zmq__.FD), read=True) else: if readers: - hubs.get_hub().schedule_call_global(0, readers[0].switch, False) + hubs.get_hub().schedule_call_global(0, readers[0][0].switch, False) elif writers: - hubs.get_hub().schedule_call_global(0, writers[0][-1].switch, False) + hubs.get_hub().schedule_call_global(0, writers[0][0].switch, False) return send_result or recv_result @@ -277,7 +333,7 @@ class Socket(__zmq__.Socket): Send as many msgs from the writers deque as possible. Wake up the greenthreads for messages that are sent. """ - writers = self.writers + writers = self._writers current = greenlet.getcurrent() hub = hubs.get_hub() super_send = super(Socket, self).send @@ -286,7 +342,7 @@ class Socket(__zmq__.Socket): result = None while writers: - msg, flags, copy, track, writer = writers[0] + writer, msg, flags, copy, track = writers[0] try: if isinstance(msg, list): r = super_send_multipart(msg, flags=flags, copy=copy, track=track) @@ -315,23 +371,51 @@ class Socket(__zmq__.Socket): # wake writer if current is not writer: hub.schedule_call_global(0, writer.switch, r) - return result - - def _xsafe_recv(self, flags=0, copy=True, track=False): - """ - A recv method that's safe to use when multiple greenthreads - are calling send, send_multipart, recv and recv_multipart on - the same socket. - """ + return result - if flags & __zmq__.NOBLOCK: - return super(Socket, self).recv(flags=flags, copy=copy, track=track) - def _xsafe_recv_multipart(self, flags=0, copy=True, track=False): + def _recv_queued(self): """ - A recv method that's safe to use when multiple greenthreads - are calling send, send_multipart, recv and recv_multipart on - the same socket. + Recv as many msgs for each of the greenthreads in the readers + deque. Wakes up the greenthreads for messages that are + received. If the received message is for the current + greenthread, returns immediately. """ - if flags & __zmq__.NOBLOCK: - return super(Socket, self).recv_multipart(flags=flags, copy=copy, track=track) + readers = self._readers + super_recv = super(Socket, self).recv + super_recv_multipart = super(Socket, self).recv_multipart + + current = greenlet.getcurrent() + hub = hubs.get_hub() + + while readers: + reader, multi, flags, copy, track = readers[0] + try: + if multi: + msg = super_recv_multipart(flags, copy, track) + else: + msg = super_recv(flags, copy, track) + + except (SystemExit, KeyboardInterrupt): + raise + except __zmq__.ZMQError, e: + if e.errno == EAGAIN: + return None + else: + # message failed to send + readers.popleft() + if current is reader: + raise + else: + hub.schedule_call_global(0, reader.throw, e) + continue + + # move to the next reader + readers.popleft() + + if current is reader: + return msg + else: + hub.schedule_call_global(0, reader.switch, msg) + + return None From 291282e78505247900dfab755908b5273e2eeea8 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Mon, 5 Sep 2011 14:38:04 -0400 Subject: [PATCH 04/41] fixed missed read events on zmq socked FD --- eventlet/green/zmq.py | 199 ++++++++++++++++++++++++------------------ tests/zmq_test.py | 76 ++++++++-------- 2 files changed, 152 insertions(+), 123 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 7e92a4f..875886c 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -46,8 +46,15 @@ class _Context(__zmq__.Context): # see http://api.zeromq.org/2-1:zmq-socket for explanation of socket types -_multi_reader_types = set([__zmq__.XREP, __zmq__.XREQ, __zmq__.SUB, __zmq__.PULL, __zmq__.PAIR]) -_multi_writer_types = set([__zmq__.XREP, __zmq__.XREQ, __zmq__.PUB, __zmq__.PUSH, __zmq__.PAIR]) +_multi_reader_types = set([__zmq__.SUB, __zmq__.PULL, __zmq__.PAIR]) +_multi_writer_types = set([__zmq__.PUB, __zmq__.PUSH, __zmq__.PAIR]) +try: + _multi_reader_types.update([__zmq__.XREP, __zmq__.XREQ]) + _multi_writer_types.update([__zmq__.XREP, __zmq__.XREQ]) +except AttributeError: + # XREP and XREQ are being renamed ROUTER and DEALER + _multi_reader_types.update([__zmq__.ROUTER, __zmq__.DEALER]) + _multi_writer_types.update([__zmq__.ROUTER, __zmq__.DEALER]) _disable_send_types = set([__zmq__.SUB, __zmq__.PULL]) _disable_recv_types = set([__zmq__.PUB, __zmq__.PUSH]) @@ -82,6 +89,12 @@ class Socket(__zmq__.Socket): self._writers = None self._readers = None + self._blocked_thread = None + self._wakeup_timer = None + + self._super_getsockopt = super(Socket, self).getsockopt + self._fd = self._super_getsockopt(__zmq__.FD) + # customize send and recv functions based on socket type if socket_type in _multi_writer_types: # support multiple greenthreads writing at the same time @@ -106,7 +119,7 @@ class Socket(__zmq__.Socket): are none it will trampoline and when coming back check for the events. """ - events = self.getsockopt(__zmq__.EVENTS) + events = self._super_getsockopt(__zmq__.EVENTS) if read and (events & __zmq__.POLLIN): return events @@ -114,8 +127,8 @@ class Socket(__zmq__.Socket): return events else: # ONLY trampoline on read events for the zmq FD - trampoline(self.getsockopt(__zmq__.FD), read=True) - return self.getsockopt(__zmq__.EVENTS) + trampoline(self._fd, read=True) + return self._super_getsockopt(__zmq__.EVENTS) def send(self, msg, flags=0, copy=True, track=False): """ @@ -156,32 +169,27 @@ class Socket(__zmq__.Socket): if e.errno != EAGAIN: raise + def getsockopt(self, option): + result = self._super_getsockopt(option) + if option == __zmq__.EVENTS: + # Getting the events causes the zmq socket to process + # events which may mean a msg can be sent or received. If + # there is a greenthread blocked and waiting for events, + # it will miss the edge-triggered read event, so wake it + # up. + if self._blocked_thread is not None: + if (self._readers and (result & __zmq__.POLLIN)) or \ + (self._writers and (result & __zmq__.POLLOUT)): + self._wake_listener() + return result + + def _send_not_supported(self, msg, flags=0, copy=True, track=False): raise __zmq__.ZMQError(__zmq__.ENOTSUP) def _recv_not_supported(self, flags=0, copy=True, track=False): raise __zmq__.ZMQError(__zmq__.ENOTSUP) - - def _xsafe_sock_wait(self, read=False, write=False): - """ - First checks if there are events in the socket, to avoid - edge trigger problems with race conditions. Then if there - are none it will trampoline and when coming back check - for the events. - """ - events = self.getsockopt(__zmq__.EVENTS) - - if read and (events & __zmq__.POLLIN): - return events - elif write and (events & __zmq__.POLLOUT): - return events - else: - # ONLY trampoline on read events for the zmq FD - trampoline(self.getsockopt(__zmq__.FD), read=True) - return self.getsockopt(__zmq__.EVENTS) - - def _xsafe_send(self, msg, flags=0, copy=True, track=False): """ A send method that's safe to use when multiple greenthreads @@ -189,9 +197,12 @@ class Socket(__zmq__.Socket): the same socket. """ if flags & __zmq__.NOBLOCK: - return super(Socket, self).send(msg, flags=flags, copy=copy, track=track) + raise __zmq__.ZMQError(__zmq__.ENOTSUP) + result = super(Socket, self).send(msg, flags=flags, copy=copy, track=track) + self._wake_listener() + return result - self._xsafe_inner_send(msg, flags, copy, track) + return self._xsafe_inner_send(msg, flags, copy, track) def _xsafe_send_multipart(self, msg_parts, flags=0, copy=True, track=False): """ @@ -203,35 +214,25 @@ class Socket(__zmq__.Socket): """ if flags & __zmq__.NOBLOCK: - return super(Socket, self).send_multipart(msg_parts, flags=flags, copy=copy, track=track) + raise __zmq__.ZMQError(__zmq__.ENOTSUP) + result = super(Socket, self).send_multipart(msg_parts, flags=flags, copy=copy, track=track) + self._wake_listener() + return result - self._xsafe_inner_send(list(msg_parts), flags, copy, track) + return self._xsafe_inner_send(list(msg_parts), flags, copy, track) def _xsafe_inner_send(self, msg, flags, copy, track): - is_listening = bool(self._writers or self._readers) - self._writers.append((greenlet.getcurrent(), msg, flags | __zmq__.NOBLOCK, copy, track)) - if is_listening: - # Other readers or writers are blocked. If this is the - # first writer, it may be possible to send immediately - if len(self._writers) == 1: - # TODO: Check EVENTS first? - result = self._send_queued() - if not self._writers: - # success! - return result - - # other readers or writers are blocked so this greenthread must wait its turn - result = hubs.get_hub().switch() - if result is False: - # msg was not yet sent, but this thread was woken up - # so that it could process the queues - return self._process_queues() - else: - # msg was sent by another greenthread + + if len(self._writers) == 1: + # no other waiting writers, may be able to send immediately + result = self._send_queued() + if not self._writers: + # received message + self._wake_listener() return result - else: - return self._process_queues() + + return self._inner_send_recv() def _xsafe_recv(self, flags=0, copy=True, track=False): """ @@ -241,7 +242,10 @@ class Socket(__zmq__.Socket): """ if flags & __zmq__.NOBLOCK: - return super(Socket, self).recv(flags=flags, copy=copy, track=track) + raise __zmq__.ZMQError(__zmq__.ENOTSUP) + msg = super(Socket, self).recv(flags=flags, copy=copy, track=track) + self._wake_listener() + return msg return self._xsafe_inner_recv(False, flags, copy, track) @@ -252,37 +256,46 @@ class Socket(__zmq__.Socket): the same socket. """ if flags & __zmq__.NOBLOCK: - return super(Socket, self).recv_multipart(flags=flags, copy=copy, track=track) + raise __zmq__.ZMQError(__zmq__.ENOTSUP) + msg = super(Socket, self).recv_multipart(flags=flags, copy=copy, track=track) + self._wake_listener() + return msg return self._xsafe_inner_recv(True, flags, copy, track) def _xsafe_inner_recv(self, multi, flags, copy, track): - is_listening = bool(self._writers or self._readers) - self._readers.append((greenlet.getcurrent(), multi, flags | __zmq__.NOBLOCK, copy, track)) - if is_listening: - # Other readers or writers are blocked. If this is the - # first reader, it may be possible to recv immediately - if len(self._readers) == 1: - # TODO: Check EVENTS first? - result = self._recv_queued() - if not self._readers: - # success! - return result - - # other readers or writers are blocked so this greenthread must wait its turn - result = hubs.get_hub().switch() - if result is False: - # msg was not yet received, but this thread was woken up - # so that it could process the queues - return self._process_queues() - else: - # msg was received + if len(self._readers) == 1: + # no other waiting readers, may be able to recv immediately + result = self._recv_queued() + if result is not None: + # received message + self._wake_listener() return result - else: - return self._process_queues() + return self._inner_send_recv() + + def _inner_send_recv(self): + if self._wake_listener(): + # Another greenthread is listening on the FD. Block this one. + result = hubs.get_hub().switch() + if result is not False: + # msg was sent or received + return result + # Send or recv has not been done, but this thread was + # woken up so that it could process the queues + + return self._process_queues() + + def _wake_listener(self): + is_listener = self._blocked_thread is not None + + if is_listener and self._wakeup_timer is None: + self._wakeup_timer = hubs.get_hub().schedule_call_global(0, self._blocked_thread.switch) + return True + + return is_listener def _process_queues(self): """ If there are readers or writers queued, this method tries @@ -290,17 +303,15 @@ class Socket(__zmq__.Socket): either in this greenthread or in another one. """ readers = self._readers writers = self._writers + current = greenlet.getcurrent() - send_result = None - recv_result = None + result = None while True: - events = self.getsockopt(__zmq__.EVENTS) try: - if readers and (events & __zmq__.POLLIN): - recv_result = self._recv_queued() - - if writers and (events & __zmq__.POLLOUT): - send_result = self._send_queued() + if readers: + result = self._recv_queued() or result + if writers: + result = self._send_queued() or result except (SystemExit, KeyboardInterrupt): raise except: @@ -312,20 +323,34 @@ class Socket(__zmq__.Socket): hubs.get_hub().schedule_call_global(0, writers[0][0].switch, False) raise + events = self._super_getsockopt(__zmq__.EVENTS) + if (readers and (events & __zmq__.POLLIN)) or \ + (writers and (events & __zmq__.POLLOUT)): + # more work to do + continue + # send and recv cannot continue right now. If there are # more readers or writers queued, either trampoline or # wake another greenthread. - current = greenlet.getcurrent() - if (readers and readers[0][0] is current) or (writers and writers[0][0] is current): + if (readers and readers[0][0] is current) or \ + (writers and writers[0][0] is current): # Only trampoline if this thread is the next reader or writer, # and ONLY trampoline on read events for zmq FDs. - trampoline(self.getsockopt(__zmq__.FD), read=True) + try: + self._blocked_thread = current + trampoline(self._fd, read=True) + finally: + if self._wakeup_timer is not None: + self._wakeup_timer.cancel() + self._wakeup_timer = None + + self._blocked_thread = None else: if readers: hubs.get_hub().schedule_call_global(0, readers[0][0].switch, False) elif writers: hubs.get_hub().schedule_call_global(0, writers[0][0].switch, False) - return send_result or recv_result + return result def _send_queued(self): diff --git a/tests/zmq_test.py b/tests/zmq_test.py index f09e872..c3392e4 100644 --- a/tests/zmq_test.py +++ b/tests/zmq_test.py @@ -242,10 +242,12 @@ got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) def test_send_during_recv(self): sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) sleep() - done = event.Event() - def slow_rx(): - self.assertEqual(sender.recv(), "done") + num_recvs = 30 + done_evts = [event.Event() for _ in range(num_recvs)] + + def slow_rx(done, msg): + self.assertEqual(sender.recv(), msg) done.send(0) def tx(): @@ -253,49 +255,51 @@ got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) while tx_i <= 1000: sender.send(str(tx_i)) tx_i += 1 - + def rx(): while True: rx_i = receiver.recv() if rx_i == "1000": - receiver.send('done') + for i in range(num_recvs): + receiver.send('done%d' % i) + sleep() return - spawn(slow_rx) + + for i in range(num_recvs): + spawn(slow_rx, done_evts[i], "done%d" % i) + spawn(tx) spawn(rx) + for i in range(num_recvs): + final_i = done_evts[i].wait() + self.assertEqual(final_i, 0) + + + # Need someway to ensure a thread is blocked on send... This isn't working + @skip_unless(zmq_supported) + def test_recv_during_send(self): + sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) + sleep() + + num_recvs = 30 + done = event.Event() + + sender.setsockopt(zmq.HWM, 10) + sender.setsockopt(zmq.SNDBUF, 10) + + receiver.setsockopt(zmq.RCVBUF, 10) + + def tx(): + tx_i = 0 + while tx_i <= 1000: + sender.send(str(tx_i)) + tx_i += 1 + done.send(0) + + spawn(tx) final_i = done.wait() self.assertEqual(final_i, 0) - # Need someway to ensure a thread is blocked on send. This method - # below uses too much memory. Try adjust watermarks or other - # socket opts? - - # @skip_unless(zmq_supported) - # def test_recv_during_send(self): - # sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) - # sleep() - # done = event.Event() - - # def tx(): - # msg = "0" * 1024 - # while True: - # sender.send(msg) - - # def rx(): - # self.assertEqual(sender.recv(), "done") - # sender_thread.kill() - # done.send(0) - - # def single_tx(): - # receiver.send("done") - - # sender_thread = spawn(tx) - # sleep() - # spawn(rx) - # spawn(single_tx) - # final_i = done.wait() - # self.assertEqual(final_i, 0) - class TestThreadedContextAccess(TestCase): """zmq's Context must be unique within a hub From 07616f69325b5d82e9a568836dc881b3bc5394eb Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Mon, 5 Sep 2011 15:09:54 -0400 Subject: [PATCH 05/41] clean up zmq process_queues loop --- eventlet/green/zmq.py | 74 ++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 875886c..fc06fb6 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -308,6 +308,9 @@ class Socket(__zmq__.Socket): result = None while True: try: + # Processing readers before writers here is arbitrary, + # but if you change the order be sure you modify the + # following code that calls getsockopt(EVENTS). if readers: result = self._recv_queued() or result if writers: @@ -323,35 +326,56 @@ class Socket(__zmq__.Socket): hubs.get_hub().schedule_call_global(0, writers[0][0].switch, False) raise - events = self._super_getsockopt(__zmq__.EVENTS) - if (readers and (events & __zmq__.POLLIN)) or \ - (writers and (events & __zmq__.POLLOUT)): - # more work to do - continue + # Above we processed all queued readers and then all + # queued writers. Each call to send or recv can cause the + # zmq to process pending events, so by calling send last + # there may now be a message waiting for a + # reader. However, if we just call recv now then further + # events may notify the socket that a pipe has room for a + # message to be send. To break this vicious cycle and + # safely call trampoline, check getsockopt(EVENTS) to + # ensure a message can't be either sent or received a + # message. + + if readers: + events = self._super_getsockopt(__zmq__.EVENTS) + if (events & __zmq__.POLLIN) or (writers and (events & __zmq__.POLLOUT)): + # more work to do + continue + + next_reader = readers[0][0] if readers else None + next_writer = writers[0][0] if writers else None + + next_thread = next_reader or next_writer # send and recv cannot continue right now. If there are # more readers or writers queued, either trampoline or # wake another greenthread. - if (readers and readers[0][0] is current) or \ - (writers and writers[0][0] is current): - # Only trampoline if this thread is the next reader or writer, - # and ONLY trampoline on read events for zmq FDs. - try: - self._blocked_thread = current - trampoline(self._fd, read=True) - finally: - if self._wakeup_timer is not None: - self._wakeup_timer.cancel() - self._wakeup_timer = None - - self._blocked_thread = None - else: - if readers: - hubs.get_hub().schedule_call_global(0, readers[0][0].switch, False) - elif writers: - hubs.get_hub().schedule_call_global(0, writers[0][0].switch, False) - return result - + if next_thread: + # Only trampoline if this thread is the next reader or writer + if next_reader is current or next_writer is current: + try: + self._blocked_thread = current + # Only trampoline on read events for zmq FDs, never write. + trampoline(self._fd, read=True) + continue + finally: + self._blocked_thread = None + # Either the fd is readable or we were woken by + # another thread. Cleanup the wakeup timer. + t = self._wakeup_timer + if t is not None: + # Important to cancel the timer so it doesn't + # spuriously wake this greenthread later on. + t.cancel() + self._wakeup_timer = None + else: + # This greenthread's work is done. Wake another to + # continue processing the queues if there is one + # blocked. This arbitrarily prefers to wake the + # next reader, but I don't think it matters which. + hubs.get_hub().schedule_call_global(0, next_thread.switch, False) + return result def _send_queued(self): """ From 6a4333203c2c0a6e6f92da3e11a6d427f663d6ad Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Mon, 5 Sep 2011 16:34:24 -0400 Subject: [PATCH 06/41] add zmq TODOs --- eventlet/green/zmq.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index fc06fb6..87719c1 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -63,8 +63,13 @@ _disable_recv_types = set([__zmq__.PUB, __zmq__.PUSH]) # TODO: # - Ensure that recv* and send* methods raise error when called on a # closed socket. They should not block. +# - Ensure that recv* and send* methods raise EFSM error when socket +# is in improper state. Avoid blocking. # - Return correct message tracker from send* methods # - Make MessageTracker.wait zmq friendly +# - What should happen to threads blocked on send/recv when socket is +# closed? + class Socket(__zmq__.Socket): """Green version of :class:`zmq.core.socket.Socket From 2f7f4302ba3adfe516fe6e0f88bb6be5ae11e565 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Mon, 5 Sep 2011 16:45:26 -0400 Subject: [PATCH 07/41] ensure any errors from zmq are raise on thread that called send/recv --- eventlet/green/zmq.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 87719c1..3d70e8e 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -412,13 +412,19 @@ class Socket(__zmq__.Socket): if e.errno == EAGAIN: return result else: - # message failed to send writers.popleft() if current is writer: raise else: hub.schedule_call_global(0, writer.throw, e) continue + except: + writers.popleft() + if current is writer: + raise + else: + hub.schedule_call_global(0, writer.throw, e) + continue # move to the next msg writers.popleft() @@ -456,13 +462,19 @@ class Socket(__zmq__.Socket): if e.errno == EAGAIN: return None else: - # message failed to send readers.popleft() if current is reader: raise else: hub.schedule_call_global(0, reader.throw, e) continue + except: + readers.popleft() + if current is reader: + raise + else: + hub.schedule_call_global(0, reader.throw, e) + continue # move to the next reader readers.popleft() From 40d4e045d496c5cd985fabd6a36b386275acbb8d Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Mon, 5 Sep 2011 16:59:14 -0400 Subject: [PATCH 08/41] avoid queuing job in zmq send/recv fast path --- eventlet/green/zmq.py | 62 ++++++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 3d70e8e..a4e03e0 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -207,7 +207,7 @@ class Socket(__zmq__.Socket): self._wake_listener() return result - return self._xsafe_inner_send(msg, flags, copy, track) + return self._xsafe_inner_send(msg, False, flags, copy, track) def _xsafe_send_multipart(self, msg_parts, flags=0, copy=True, track=False): """ @@ -224,19 +224,33 @@ class Socket(__zmq__.Socket): self._wake_listener() return result - return self._xsafe_inner_send(list(msg_parts), flags, copy, track) + return self._xsafe_inner_send(msg_parts, True, flags, copy, track) - def _xsafe_inner_send(self, msg, flags, copy, track): - self._writers.append((greenlet.getcurrent(), msg, flags | __zmq__.NOBLOCK, copy, track)) + def _xsafe_inner_send(self, msg, multi, flags, copy, track): + flags |= __zmq__.NOBLOCK + if not self._writers: + # no other waiting writers, may be able to send + # immediately. This is the fast path. + try: + if multi: + r = super(Socket, self).send_multipart( + msg, flags=flags, copy=copy, track=track) + else: + r = super(Socket, self).send( + msg, flags=flags, copy=copy, track=track) - if len(self._writers) == 1: - # no other waiting writers, may be able to send immediately - result = self._send_queued() - if not self._writers: - # received message self._wake_listener() - return result + return r + except __zmq__.ZMQError, e: + if e.errno != EAGAIN: + raise + # copy msg lists so they can't be modified by caller + if multi: + msg = list(msg) + + # queue msg to be sent later + self._writers.append((greenlet.getcurrent(), multi, msg, flags, copy, track)) return self._inner_send_recv() def _xsafe_recv(self, flags=0, copy=True, track=False): @@ -269,16 +283,26 @@ class Socket(__zmq__.Socket): return self._xsafe_inner_recv(True, flags, copy, track) def _xsafe_inner_recv(self, multi, flags, copy, track): - self._readers.append((greenlet.getcurrent(), multi, flags | __zmq__.NOBLOCK, copy, track)) + flags |= __zmq__.NOBLOCK + if not self._readers: + # no other waiting readers, may be able to recv + # immediately. This is the fast path. + try: + if multi: + msg = super(Socket, self).recv_multipart( + flags=flags, copy=copy, track=track) + else: + msg = super(Socket, self).recv( + flags=flags, copy=copy, track=track) - if len(self._readers) == 1: - # no other waiting readers, may be able to recv immediately - result = self._recv_queued() - if result is not None: - # received message self._wake_listener() - return result + return msg + except __zmq__.ZMQError, e: + if e.errno != EAGAIN: + raise + # queue recv for later + self._readers.append((greenlet.getcurrent(), multi, flags, copy, track)) return self._inner_send_recv() def _inner_send_recv(self): @@ -396,9 +420,9 @@ class Socket(__zmq__.Socket): result = None while writers: - writer, msg, flags, copy, track = writers[0] + writer, multi, msg, flags, copy, track = writers[0] try: - if isinstance(msg, list): + if multi: r = super_send_multipart(msg, flags=flags, copy=copy, track=track) else: r = super_send(msg, flags=flags, copy=copy, track=track) From 3f2805d12ba53083c2c7eb7cb01d68f645223b2a Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Mon, 5 Sep 2011 17:34:33 -0400 Subject: [PATCH 09/41] use interruptible trampoline for zmq REQ and REP sockets. Prefer positional args when have the choice. --- eventlet/green/zmq.py | 126 ++++++++++++++++++++---------------------- 1 file changed, 60 insertions(+), 66 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index a4e03e0..ab936f2 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -63,8 +63,6 @@ _disable_recv_types = set([__zmq__.PUB, __zmq__.PUSH]) # TODO: # - Ensure that recv* and send* methods raise error when called on a # closed socket. They should not block. -# - Ensure that recv* and send* methods raise EFSM error when socket -# is in improper state. Avoid blocking. # - Return correct message tracker from send* methods # - Make MessageTracker.wait zmq friendly # - What should happen to threads blocked on send/recv when socket is @@ -117,23 +115,44 @@ class Socket(__zmq__.Socket): elif socket_type in _disable_recv_types: self.recv = self.recv_multipart = self._recv_not_supported - def _sock_wait(self, read=False, write=False): - """ - First checks if there are events in the socket, to avoid - edge trigger problems with race conditions. Then if there - are none it will trampoline and when coming back check - for the events. - """ - events = self._super_getsockopt(__zmq__.EVENTS) + def _trampoline(self): + """Wait for events on the zmq socket. After this method + returns it is still possible that send and recv will return + EAGAIN. - if read and (events & __zmq__.POLLIN): - return events - elif write and (events & __zmq__.POLLOUT): - return events - else: - # ONLY trampoline on read events for the zmq FD + Because the zmq FD is edge triggered, any call that causes the + zmq socket to process its events must wake the greenthread + that called trampoline by calling _wake_listener in case it + missed the event.""" + try: + self._blocked_thread = greenlet.getcurrent() + # Only trampoline on read events for zmq FDs, never write. trampoline(self._fd, read=True) - return self._super_getsockopt(__zmq__.EVENTS) + finally: + self._blocked_thread = None + # Either the fd is readable or we were woken by + # another thread. Cleanup the wakeup timer. + t = self._wakeup_timer + if t is not None: + # Important to cancel the timer so it doesn't + # spuriously wake this greenthread later on. + t.cancel() + self._wakeup_timer = None + + def _wake_listener(self): + """If a thread has called trampoline, wake it up. This can + safely be called multiple times and will have no effect if the + thread has already been woken up. + + Returns True if there is a listener thread that called + trampoline, False if not.""" + is_listener = self._blocked_thread is not None + + if is_listener and self._wakeup_timer is None: + self._wakeup_timer = hubs.get_hub().schedule_call_global(0, self._blocked_thread.switch) + return True + + return is_listener def send(self, msg, flags=0, copy=True, track=False): """ @@ -142,17 +161,17 @@ class Socket(__zmq__.Socket): called in real code. """ if flags & __zmq__.NOBLOCK: - super(Socket, self).send(msg, flags=flags, copy=copy, track=track) - return + return super(Socket, self).send(msg, flags, copy, track) flags |= __zmq__.NOBLOCK while True: try: - self._sock_wait(write=True) - return super(Socket, self).send(msg, flags=flags, copy=copy, track=track) + return super(Socket, self).send(msg, flags, copy, track) except __zmq__.ZMQError, e: - if e.errno != EAGAIN: + if e.errno == EAGAIN: + self._trampoline() + else: raise def recv(self, flags=0, copy=True, track=False): @@ -162,16 +181,17 @@ class Socket(__zmq__.Socket): called in real code. """ if flags & __zmq__.NOBLOCK: - return super(Socket, self).recv(flags=flags, copy=copy, track=track) + return super(Socket, self).recv(flags, copy, track) flags |= __zmq__.NOBLOCK while True: try: - self._sock_wait(read=True) - return super(Socket, self).recv(flags=flags, copy=copy, track=track) + return super(Socket, self).recv(flags, copy, track) except __zmq__.ZMQError, e: - if e.errno != EAGAIN: + if e.errno == EAGAIN: + self._trampoline() + else: raise def getsockopt(self, option): @@ -189,10 +209,10 @@ class Socket(__zmq__.Socket): return result - def _send_not_supported(self, msg, flags=0, copy=True, track=False): + def _send_not_supported(self, msg, flags, copy, track): raise __zmq__.ZMQError(__zmq__.ENOTSUP) - def _recv_not_supported(self, flags=0, copy=True, track=False): + def _recv_not_supported(self, flags, copy, track): raise __zmq__.ZMQError(__zmq__.ENOTSUP) def _xsafe_send(self, msg, flags=0, copy=True, track=False): @@ -203,7 +223,7 @@ class Socket(__zmq__.Socket): """ if flags & __zmq__.NOBLOCK: raise __zmq__.ZMQError(__zmq__.ENOTSUP) - result = super(Socket, self).send(msg, flags=flags, copy=copy, track=track) + result = super(Socket, self).send(msg, flags, copy, track) self._wake_listener() return result @@ -220,7 +240,7 @@ class Socket(__zmq__.Socket): if flags & __zmq__.NOBLOCK: raise __zmq__.ZMQError(__zmq__.ENOTSUP) - result = super(Socket, self).send_multipart(msg_parts, flags=flags, copy=copy, track=track) + result = super(Socket, self).send_multipart(msg_parts, flags, copy, track) self._wake_listener() return result @@ -233,11 +253,9 @@ class Socket(__zmq__.Socket): # immediately. This is the fast path. try: if multi: - r = super(Socket, self).send_multipart( - msg, flags=flags, copy=copy, track=track) + r = super(Socket, self).send_multipart(msg, flags, copy, track) else: - r = super(Socket, self).send( - msg, flags=flags, copy=copy, track=track) + r = super(Socket, self).send( msg, flags, copy, track) self._wake_listener() return r @@ -262,7 +280,7 @@ class Socket(__zmq__.Socket): if flags & __zmq__.NOBLOCK: raise __zmq__.ZMQError(__zmq__.ENOTSUP) - msg = super(Socket, self).recv(flags=flags, copy=copy, track=track) + msg = super(Socket, self).recv(flags, copy, track) self._wake_listener() return msg @@ -276,7 +294,7 @@ class Socket(__zmq__.Socket): """ if flags & __zmq__.NOBLOCK: raise __zmq__.ZMQError(__zmq__.ENOTSUP) - msg = super(Socket, self).recv_multipart(flags=flags, copy=copy, track=track) + msg = super(Socket, self).recv_multipart(flags, copy, track) self._wake_listener() return msg @@ -289,11 +307,9 @@ class Socket(__zmq__.Socket): # immediately. This is the fast path. try: if multi: - msg = super(Socket, self).recv_multipart( - flags=flags, copy=copy, track=track) + msg = super(Socket, self).recv_multipart(flags, copy, track) else: - msg = super(Socket, self).recv( - flags=flags, copy=copy, track=track) + msg = super(Socket, self).recv(flags, copy, track) self._wake_listener() return msg @@ -317,15 +333,6 @@ class Socket(__zmq__.Socket): return self._process_queues() - def _wake_listener(self): - is_listener = self._blocked_thread is not None - - if is_listener and self._wakeup_timer is None: - self._wakeup_timer = hubs.get_hub().schedule_call_global(0, self._blocked_thread.switch) - return True - - return is_listener - def _process_queues(self): """ If there are readers or writers queued, this method tries to recv or send messages and ensures processing continues @@ -383,21 +390,8 @@ class Socket(__zmq__.Socket): if next_thread: # Only trampoline if this thread is the next reader or writer if next_reader is current or next_writer is current: - try: - self._blocked_thread = current - # Only trampoline on read events for zmq FDs, never write. - trampoline(self._fd, read=True) - continue - finally: - self._blocked_thread = None - # Either the fd is readable or we were woken by - # another thread. Cleanup the wakeup timer. - t = self._wakeup_timer - if t is not None: - # Important to cancel the timer so it doesn't - # spuriously wake this greenthread later on. - t.cancel() - self._wakeup_timer = None + self._trampoline() + continue else: # This greenthread's work is done. Wake another to # continue processing the queues if there is one @@ -423,9 +417,9 @@ class Socket(__zmq__.Socket): writer, multi, msg, flags, copy, track = writers[0] try: if multi: - r = super_send_multipart(msg, flags=flags, copy=copy, track=track) + r = super_send_multipart(msg, flags, copy, track) else: - r = super_send(msg, flags=flags, copy=copy, track=track) + r = super_send(msg, flags, copy, track) # remember this thread's result if current is writer: From e9830d021a6d01dc3d89960b5e2e86ea601ac284 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Mon, 5 Sep 2011 18:19:01 -0400 Subject: [PATCH 10/41] Fix up comments in zmq. Take __name__ and __doc__ for methods from the real Socket class. Also removes some raise statements accidently left in --- eventlet/green/zmq.py | 76 +++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index ab936f2..f7cf487 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -68,6 +68,12 @@ _disable_recv_types = set([__zmq__.PUB, __zmq__.PUSH]) # - What should happen to threads blocked on send/recv when socket is # closed? +def _wraps(source_fn): + def wrapper(dest_fn): + dest_fn.__name__ = source_fn.__name__ + dest_fn.__doc__ = source_fn.__doc__ + return dest_fn + return wrapper class Socket(__zmq__.Socket): """Green version of :class:`zmq.core.socket.Socket @@ -75,6 +81,7 @@ class Socket(__zmq__.Socket): The following two methods are always overridden: * send * recv + * getsockopt To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or recieving is deferred to the hub (using :func:`eventlet.hubs.trampoline`) if a ``zmq.EAGAIN`` (retry) error is raised @@ -84,7 +91,6 @@ class Socket(__zmq__.Socket): overridden: * send_multipart * recv_multipart - """ def __init__(self, context, socket_type): @@ -123,7 +129,8 @@ class Socket(__zmq__.Socket): Because the zmq FD is edge triggered, any call that causes the zmq socket to process its events must wake the greenthread that called trampoline by calling _wake_listener in case it - missed the event.""" + missed the event. + """ try: self._blocked_thread = greenlet.getcurrent() # Only trampoline on read events for zmq FDs, never write. @@ -145,7 +152,8 @@ class Socket(__zmq__.Socket): thread has already been woken up. Returns True if there is a listener thread that called - trampoline, False if not.""" + trampoline, False if not. + """ is_listener = self._blocked_thread is not None if is_listener and self._wakeup_timer is None: @@ -154,11 +162,11 @@ class Socket(__zmq__.Socket): return is_listener + @_wraps(__zmq__.Socket.send) def send(self, msg, flags=0, copy=True, track=False): - """ - Override this instead of the internal _send_* methods - since those change and it's not clear when/how they're - called in real code. + """Send method used by REP and REQ sockets. The lock-step + send->recv->send->recv restriction of these sockets makes this + implementation simple. """ if flags & __zmq__.NOBLOCK: return super(Socket, self).send(msg, flags, copy, track) @@ -174,11 +182,11 @@ class Socket(__zmq__.Socket): else: raise + @_wraps(__zmq__.Socket.recv) def recv(self, flags=0, copy=True, track=False): - """ - Override this instead of the internal _recv_* methods - since those change and it's not clear when/how they're - called in real code. + """Recv method used by REP and REQ sockets. The lock-step + send->recv->send->recv restriction of these sockets makes this + implementation simple. """ if flags & __zmq__.NOBLOCK: return super(Socket, self).recv(flags, copy, track) @@ -194,6 +202,7 @@ class Socket(__zmq__.Socket): else: raise + @_wraps(__zmq__.Socket.getsockopt) def getsockopt(self, option): result = self._super_getsockopt(option) if option == __zmq__.EVENTS: @@ -207,7 +216,6 @@ class Socket(__zmq__.Socket): (self._writers and (result & __zmq__.POLLOUT)): self._wake_listener() return result - def _send_not_supported(self, msg, flags, copy, track): raise __zmq__.ZMQError(__zmq__.ENOTSUP) @@ -215,31 +223,26 @@ class Socket(__zmq__.Socket): def _recv_not_supported(self, flags, copy, track): raise __zmq__.ZMQError(__zmq__.ENOTSUP) + @_wraps(__zmq__.Socket.send) def _xsafe_send(self, msg, flags=0, copy=True, track=False): - """ - A send method that's safe to use when multiple greenthreads + """A send method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. """ if flags & __zmq__.NOBLOCK: - raise __zmq__.ZMQError(__zmq__.ENOTSUP) result = super(Socket, self).send(msg, flags, copy, track) self._wake_listener() return result return self._xsafe_inner_send(msg, False, flags, copy, track) + @_wraps(__zmq__.Socket.send_multipart) def _xsafe_send_multipart(self, msg_parts, flags=0, copy=True, track=False): - """ - A send_multipart method that's safe to use when multiple + """A send_multipart method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. - - Ensure multipart messages are not interleaved. """ - if flags & __zmq__.NOBLOCK: - raise __zmq__.ZMQError(__zmq__.ENOTSUP) result = super(Socket, self).send_multipart(msg_parts, flags, copy, track) self._wake_listener() return result @@ -271,29 +274,26 @@ class Socket(__zmq__.Socket): self._writers.append((greenlet.getcurrent(), multi, msg, flags, copy, track)) return self._inner_send_recv() + @_wraps(__zmq__.Socket.recv) def _xsafe_recv(self, flags=0, copy=True, track=False): - """ - A recv method that's safe to use when multiple greenthreads + """A recv method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. """ - if flags & __zmq__.NOBLOCK: - raise __zmq__.ZMQError(__zmq__.ENOTSUP) msg = super(Socket, self).recv(flags, copy, track) self._wake_listener() return msg return self._xsafe_inner_recv(False, flags, copy, track) + @_wraps(__zmq__.Socket.recv_multipart) def _xsafe_recv_multipart(self, flags=0, copy=True, track=False): - """ - A recv method that's safe to use when multiple greenthreads - are calling send, send_multipart, recv and recv_multipart on - the same socket. + """A recv_multipart method that's safe to use when multiple + greenthreads are calling send, send_multipart, recv and + recv_multipart on the same socket. """ if flags & __zmq__.NOBLOCK: - raise __zmq__.ZMQError(__zmq__.ENOTSUP) msg = super(Socket, self).recv_multipart(flags, copy, track) self._wake_listener() return msg @@ -334,9 +334,10 @@ class Socket(__zmq__.Socket): return self._process_queues() def _process_queues(self): - """ If there are readers or writers queued, this method tries + """If there are readers or writers queued, this method tries to recv or send messages and ensures processing continues - either in this greenthread or in another one. """ + either in this greenthread or in another one. + """ readers = self._readers writers = self._writers current = greenlet.getcurrent() @@ -401,9 +402,8 @@ class Socket(__zmq__.Socket): return result def _send_queued(self): - """ - Send as many msgs from the writers deque as possible. Wake up - the greenthreads for messages that are sent. + """Send as many msgs from the writers deque as possible. Wake + up the greenthreads for messages that are sent. """ writers = self._writers current = greenlet.getcurrent() @@ -451,11 +451,9 @@ class Socket(__zmq__.Socket): hub.schedule_call_global(0, writer.switch, r) return result - def _recv_queued(self): - """ - Recv as many msgs for each of the greenthreads in the readers - deque. Wakes up the greenthreads for messages that are + """Recv as many msgs for each of the greenthreads in the + readers deque. Wakes up the greenthreads for messages that are received. If the received message is for the current greenthread, returns immediately. """ From 5e5d02d47087982711bfd5635b0244f37a335b23 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Mon, 5 Sep 2011 20:22:52 -0400 Subject: [PATCH 11/41] change how zmq socket send and recv operations are customized --- eventlet/green/zmq.py | 82 ++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index f7cf487..cc56fde 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -10,6 +10,7 @@ __patched__ = ['Context', 'Socket'] slurp_properties(__zmq__, globals(), ignore=__patched__) from collections import deque +from types import MethodType def Context(io_threads=1): """Factory function replacement for :class:`zmq.core.context.Context` @@ -45,21 +46,6 @@ class _Context(__zmq__.Context): return Socket(self, socket_type) -# see http://api.zeromq.org/2-1:zmq-socket for explanation of socket types -_multi_reader_types = set([__zmq__.SUB, __zmq__.PULL, __zmq__.PAIR]) -_multi_writer_types = set([__zmq__.PUB, __zmq__.PUSH, __zmq__.PAIR]) -try: - _multi_reader_types.update([__zmq__.XREP, __zmq__.XREQ]) - _multi_writer_types.update([__zmq__.XREP, __zmq__.XREQ]) -except AttributeError: - # XREP and XREQ are being renamed ROUTER and DEALER - _multi_reader_types.update([__zmq__.ROUTER, __zmq__.DEALER]) - _multi_writer_types.update([__zmq__.ROUTER, __zmq__.DEALER]) - -_disable_send_types = set([__zmq__.SUB, __zmq__.PULL]) -_disable_recv_types = set([__zmq__.PUB, __zmq__.PUSH]) - - # TODO: # - Ensure that recv* and send* methods raise error when called on a # closed socket. They should not block. @@ -69,6 +55,9 @@ _disable_recv_types = set([__zmq__.PUB, __zmq__.PUSH]) # closed? def _wraps(source_fn): + """A decorator that copies the __name__ and __doc__ from the given + function + """ def wrapper(dest_fn): dest_fn.__name__ = source_fn.__name__ dest_fn.__doc__ = source_fn.__doc__ @@ -96,30 +85,31 @@ class Socket(__zmq__.Socket): def __init__(self, context, socket_type): super(Socket, self).__init__(context, socket_type) - self._writers = None - self._readers = None self._blocked_thread = None self._wakeup_timer = None self._super_getsockopt = super(Socket, self).getsockopt self._fd = self._super_getsockopt(__zmq__.FD) - # customize send and recv functions based on socket type - if socket_type in _multi_writer_types: - # support multiple greenthreads writing at the same time - self._writers = deque() - self.send = self._xsafe_send - self.send_multipart = self._xsafe_send_multipart - elif socket_type in _disable_send_types: - self.send = self.send_multipart = self._send_not_supported + # customize send and recv methods based on socket type + ops = self._eventlet_ops.get(socket_type) + if ops: + self._writers = None + self._readers = None + send, msend, recv, mrecv = ops + if send: + self._writers = deque() + self.send = MethodType(send, self, Socket) + self.send_multipart = MethodType(msend, self, Socket) + else: + self.send = self.send_multipart = self._send_not_supported - if socket_type in _multi_reader_types: - # support multiple greenthreads reading at the same time - self._readers = deque() - self.recv = self._xsafe_recv - self.recv_multipart = self._xsafe_recv_multipart - elif socket_type in _disable_recv_types: - self.recv = self.recv_multipart = self._recv_not_supported + if recv: + self._readers = deque() + self.recv = MethodType(recv, self, Socket) + self.recv_multipart = MethodType(mrecv, self, Socket) + else: + self.recv = self.recv_multipart = self._send_not_supported def _trampoline(self): """Wait for events on the zmq socket. After this method @@ -501,3 +491,31 @@ class Socket(__zmq__.Socket): hub.schedule_call_global(0, reader.switch, msg) return None + + + # The behavior of the send and recv methods depends on the socket + # type. See http://api.zeromq.org/2-1:zmq-socket for explanation + # of socket types. For the green Socket, our main concern is + # supporting calling send or recv from multiple greenthreads when + # it makes sense for the socket type. + _send_only_ops = (_xsafe_send, _xsafe_send_multipart, None, None) + _recv_only_ops = (None, None, _xsafe_recv, _xsafe_recv_multipart) + _full_ops = (_xsafe_send, _xsafe_send_multipart, _xsafe_recv, _xsafe_recv_multipart) + + _eventlet_ops = { + __zmq__.PUB: _send_only_ops, + __zmq__.SUB: _recv_only_ops, + + __zmq__.PUSH: _send_only_ops, + __zmq__.PULL: _recv_only_ops, + + __zmq__.PAIR: _full_ops + } + + try: + _eventlet_ops[__zmq__.XREP] = _full_ops + _eventlet_ops[__zmq__.XREQ] = _full_ops + except AttributeError: + # XREP and XREQ are being renamed ROUTER and DEALER + _eventlet_ops[__zmq__.ROUTER] = _full_ops + _eventlet_ops[__zmq__.DEALER] = _full_ops From 0841091bf2dcd7fc11724a0784fd7d5808387204 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Mon, 5 Sep 2011 20:41:52 -0400 Subject: [PATCH 12/41] Remove pointless caching of attributes in zmq.Socket --- eventlet/green/zmq.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index cc56fde..3e2956a 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -88,9 +88,6 @@ class Socket(__zmq__.Socket): self._blocked_thread = None self._wakeup_timer = None - self._super_getsockopt = super(Socket, self).getsockopt - self._fd = self._super_getsockopt(__zmq__.FD) - # customize send and recv methods based on socket type ops = self._eventlet_ops.get(socket_type) if ops: @@ -124,7 +121,7 @@ class Socket(__zmq__.Socket): try: self._blocked_thread = greenlet.getcurrent() # Only trampoline on read events for zmq FDs, never write. - trampoline(self._fd, read=True) + trampoline(self.getsockopt(__zmq__.FD), read=True) finally: self._blocked_thread = None # Either the fd is readable or we were woken by @@ -194,7 +191,7 @@ class Socket(__zmq__.Socket): @_wraps(__zmq__.Socket.getsockopt) def getsockopt(self, option): - result = self._super_getsockopt(option) + result = super(Socket, self).getsockopt(option) if option == __zmq__.EVENTS: # Getting the events causes the zmq socket to process # events which may mean a msg can be sent or received. If @@ -365,7 +362,7 @@ class Socket(__zmq__.Socket): # message. if readers: - events = self._super_getsockopt(__zmq__.EVENTS) + events = self.getsockopt(__zmq__.EVENTS) if (events & __zmq__.POLLIN) or (writers and (events & __zmq__.POLLOUT)): # more work to do continue From d4748e8ea394f69d0b08a7fc4605e9069f4be35e Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Mon, 5 Sep 2011 21:42:24 -0400 Subject: [PATCH 13/41] zmq additional comment --- eventlet/green/zmq.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 3e2956a..739d5e5 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -390,7 +390,10 @@ class Socket(__zmq__.Socket): def _send_queued(self): """Send as many msgs from the writers deque as possible. Wake - up the greenthreads for messages that are sent. + up the greenthreads for messages that are sent. Continue + sending messages even after this greenthread has sent its own + message because that seems like the best way to increase + throughput but that is an untested assumption. """ writers = self._writers current = greenlet.getcurrent() From 95c2594b1838357f6b83c2189b935f3a39004e3c Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Fri, 9 Sep 2011 13:18:27 -0400 Subject: [PATCH 14/41] check EVENTS before waking up listener --- eventlet/green/zmq.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 739d5e5..4ff9a23 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -144,7 +144,7 @@ class Socket(__zmq__.Socket): is_listener = self._blocked_thread is not None if is_listener and self._wakeup_timer is None: - self._wakeup_timer = hubs.get_hub().schedule_call_global(0, self._blocked_thread.switch) + self.getsockopt(__zmq__.EVENTS) # triggers the wakeup return True return is_listener @@ -198,10 +198,10 @@ class Socket(__zmq__.Socket): # there is a greenthread blocked and waiting for events, # it will miss the edge-triggered read event, so wake it # up. - if self._blocked_thread is not None: + if self._blocked_thread is not None and self._wakeup_timer is None: if (self._readers and (result & __zmq__.POLLIN)) or \ (self._writers and (result & __zmq__.POLLOUT)): - self._wake_listener() + self._wakeup_timer = hubs.get_hub().schedule_call_global(0, self._blocked_thread.switch) return result def _send_not_supported(self, msg, flags, copy, track): From 33e4b782a92e02f7fe73eedb82fc20756ddcee86 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Wed, 14 Sep 2011 12:48:47 -0400 Subject: [PATCH 15/41] rename wakeup_timer -> wakeupper to avoid confusion. It's not really a timer. --- eventlet/green/zmq.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 4ff9a23..f5010f2 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -86,7 +86,7 @@ class Socket(__zmq__.Socket): super(Socket, self).__init__(context, socket_type) self._blocked_thread = None - self._wakeup_timer = None + self._wakeupper = None # customize send and recv methods based on socket type ops = self._eventlet_ops.get(socket_type) @@ -125,13 +125,13 @@ class Socket(__zmq__.Socket): finally: self._blocked_thread = None # Either the fd is readable or we were woken by - # another thread. Cleanup the wakeup timer. - t = self._wakeup_timer + # another thread. Cleanup the wakeup task. + t = self._wakeupper if t is not None: - # Important to cancel the timer so it doesn't + # Important to cancel the wakeup task so it doesn't # spuriously wake this greenthread later on. t.cancel() - self._wakeup_timer = None + self._wakeupper = None def _wake_listener(self): """If a thread has called trampoline, wake it up. This can @@ -143,7 +143,7 @@ class Socket(__zmq__.Socket): """ is_listener = self._blocked_thread is not None - if is_listener and self._wakeup_timer is None: + if is_listener and self._wakeupper is None: self.getsockopt(__zmq__.EVENTS) # triggers the wakeup return True @@ -198,10 +198,10 @@ class Socket(__zmq__.Socket): # there is a greenthread blocked and waiting for events, # it will miss the edge-triggered read event, so wake it # up. - if self._blocked_thread is not None and self._wakeup_timer is None: + if self._blocked_thread is not None and self._wakeupper is None: if (self._readers and (result & __zmq__.POLLIN)) or \ (self._writers and (result & __zmq__.POLLOUT)): - self._wakeup_timer = hubs.get_hub().schedule_call_global(0, self._blocked_thread.switch) + self._wakeupper = hubs.get_hub().schedule_call_global(0, self._blocked_thread.switch) return result def _send_not_supported(self, msg, flags, copy, track): From 03e4404b19138438c8f2bc1b802da4b37ed057a1 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Wed, 14 Sep 2011 16:22:12 -0400 Subject: [PATCH 16/41] simplify code for supporting multiple 0mq senders and receivers --- eventlet/green/zmq.py | 471 ++++++++++++++---------------------------- 1 file changed, 156 insertions(+), 315 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index f5010f2..e59e540 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -12,6 +12,91 @@ slurp_properties(__zmq__, globals(), ignore=__patched__) from collections import deque from types import MethodType +class _QueueLock(object): + """A Lock that can be acquired by at most one thread. Any other + thread calling acquire will be blocked in a queue. When release + is called, the threads are awoken in the order they blocked, + one at a time. This lock can be required recursively by the same + thread.""" + def __init__(self): + self._waiters = deque() + self._count = 0 + self._holder = None + + def __nonzero__(self): + return self._count + + def __enter__(self): + self.acquire() + + def __exit__(self, type, value, traceback): + self.release() + + def acquire(self): + current = greenlet.getcurrent() + if (self._waiters or self._count > 0) and self._holder is not current: + # block until lock is free + self._waiters.append(current) + hubs.get_hub().switch() + w = self._waiters.popleft() + + assert w is current, 'Waiting threads woken out of order' + assert self._count == 0, 'After waking a thread, the lock must be unacquired' + + self._holder = current + self._count += 1 + + def release(self): + assert self._count > 0, 'Cannot release unacquired lock' + + self._count -= 1 + if self._count == 0: + self._holder = None + if self._waiters: + # wake next + hubs.get_hub().schedule_call_global(0, self._waiters[0].switch) + +class _SimpleEvent(object): + """Represents a possibly blocked thread which may be blocked + inside this class' block method or inside a trampoline call. In + either case, the threads can be awoken by calling wake(). Wake() + can be called multiple times and all but the first call will have + no effect.""" + + def __init__(self): + self._blocked_thread = None + self._wakeupper = None + + def __nonzero__(self): + return self._blocked_thread is not None + + def block(self): + with self: + hubs.get_hub().switch() + + def __enter__(self): + assert self._blocked_thread is None, 'Cannot block more than one thread on one SimpleEvent' + self._blocked_thread = greenlet.getcurrent() + + def __exit__(self, type, value, traceback): + self._blocked_thread = None + # cleanup the wakeup task + if self._wakeupper is not None: + # Important to cancel the wakeup task so it doesn't + # spuriously wake this greenthread later on. + self._wakeupper.cancel() + self._wakeupper = None + + def wake(self): + """Schedules the blocked thread to be awoken and return + True. If wake has already been called or if there is no + blocked thread, then this call has no effect and returns + False.""" + if self._blocked_thread is not None and self._wakeupper is None: + self._wakeupper = hubs.get_hub().schedule_call_global(0, self._blocked_thread.switch) + return True + return False + def Context(io_threads=1): """Factory function replacement for :class:`zmq.core.context.Context` @@ -74,80 +159,56 @@ class Socket(__zmq__.Socket): To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or recieving is deferred to the hub (using :func:`eventlet.hubs.trampoline`) if a ``zmq.EAGAIN`` (retry) error is raised - - For some socket types, where multiple greenthreads could be - calling send or recv at the same time, these methods are also - overridden: - * send_multipart - * recv_multipart """ def __init__(self, context, socket_type): super(Socket, self).__init__(context, socket_type) - self._blocked_thread = None - self._wakeupper = None + self._in_trampoline = False + self._send_event = _SimpleEvent() + self._recv_event = _SimpleEvent() # customize send and recv methods based on socket type ops = self._eventlet_ops.get(socket_type) if ops: - self._writers = None - self._readers = None - send, msend, recv, mrecv = ops + self._send_lock = None + self._recv_lock = None + send, recv = ops if send: - self._writers = deque() + self._send_lock = _QueueLock() self.send = MethodType(send, self, Socket) - self.send_multipart = MethodType(msend, self, Socket) else: - self.send = self.send_multipart = self._send_not_supported + self.send = self._send_not_supported if recv: - self._readers = deque() + self._recv_lock = _QueueLock() self.recv = MethodType(recv, self, Socket) - self.recv_multipart = MethodType(mrecv, self, Socket) else: - self.recv = self.recv_multipart = self._send_not_supported + self.recv = self._send_not_supported - def _trampoline(self): + def _trampoline(self, is_send): """Wait for events on the zmq socket. After this method returns it is still possible that send and recv will return EAGAIN. - Because the zmq FD is edge triggered, any call that causes the - zmq socket to process its events must wake the greenthread - that called trampoline by calling _wake_listener in case it - missed the event. + This supports being called by two separate greenthreads, a + sender and a receiver, but only the first caller will actually + call eventlet's trampoline method. The second thread will + still block. """ - try: - self._blocked_thread = greenlet.getcurrent() - # Only trampoline on read events for zmq FDs, never write. - trampoline(self.getsockopt(__zmq__.FD), read=True) - finally: - self._blocked_thread = None - # Either the fd is readable or we were woken by - # another thread. Cleanup the wakeup task. - t = self._wakeupper - if t is not None: - # Important to cancel the wakeup task so it doesn't - # spuriously wake this greenthread later on. - t.cancel() - self._wakeupper = None - def _wake_listener(self): - """If a thread has called trampoline, wake it up. This can - safely be called multiple times and will have no effect if the - thread has already been woken up. - - Returns True if there is a listener thread that called - trampoline, False if not. - """ - is_listener = self._blocked_thread is not None - - if is_listener and self._wakeupper is None: - self.getsockopt(__zmq__.EVENTS) # triggers the wakeup - return True - - return is_listener + evt = self._send_event if is_send else self._recv_event + if self._in_trampoline: + # Already a thread blocked in trampoline. + evt.block() + else: + try: + self._in_trampoline = True + with evt: + # Only trampoline on read events for zmq FDs, never write. + trampoline(self.getsockopt(__zmq__.FD), read=True) + finally: + self._in_trampoline = False @_wraps(__zmq__.Socket.send) def send(self, msg, flags=0, copy=True, track=False): @@ -162,10 +223,10 @@ class Socket(__zmq__.Socket): while True: try: - return super(Socket, self).send(msg, flags, copy, track) + return super(Socket, self).send(msg, flags, copy, track) except __zmq__.ZMQError, e: if e.errno == EAGAIN: - self._trampoline() + self._trampoline(True) else: raise @@ -185,7 +246,7 @@ class Socket(__zmq__.Socket): return super(Socket, self).recv(flags, copy, track) except __zmq__.ZMQError, e: if e.errno == EAGAIN: - self._trampoline() + self._trampoline(False) else: raise @@ -198,10 +259,11 @@ class Socket(__zmq__.Socket): # there is a greenthread blocked and waiting for events, # it will miss the edge-triggered read event, so wake it # up. - if self._blocked_thread is not None and self._wakeupper is None: - if (self._readers and (result & __zmq__.POLLIN)) or \ - (self._writers and (result & __zmq__.POLLOUT)): - self._wakeupper = hubs.get_hub().schedule_call_global(0, self._blocked_thread.switch) + if self._send_evt and (result & __zmq__.POLLOUT): + self._send_evt.wake() + + if self._recv_evt and (result & __zmq__.POLLIN): + self._recv_evt.wake() return result def _send_not_supported(self, msg, flags, copy, track): @@ -221,45 +283,27 @@ class Socket(__zmq__.Socket): self._wake_listener() return result - return self._xsafe_inner_send(msg, False, flags, copy, track) - - @_wraps(__zmq__.Socket.send_multipart) - def _xsafe_send_multipart(self, msg_parts, flags=0, copy=True, track=False): - """A send_multipart method that's safe to use when multiple - greenthreads are calling send, send_multipart, recv and - recv_multipart on the same socket. - """ - if flags & __zmq__.NOBLOCK: - result = super(Socket, self).send_multipart(msg_parts, flags, copy, track) - self._wake_listener() - return result - - return self._xsafe_inner_send(msg_parts, True, flags, copy, track) - - def _xsafe_inner_send(self, msg, multi, flags, copy, track): + # TODO: pyzmq will copy the message buffer and create Message + # objects under some circumstances. We could do that work here + # once to avoid doing it every time the send is retried. flags |= __zmq__.NOBLOCK - if not self._writers: - # no other waiting writers, may be able to send - # immediately. This is the fast path. - try: - if multi: - r = super(Socket, self).send_multipart(msg, flags, copy, track) - else: - r = super(Socket, self).send( msg, flags, copy, track) + with self._send_lock: + while True: + try: + try: + return super(Socket, self).send(msg, flags, copy, track) + finally: - self._wake_listener() - return r - except __zmq__.ZMQError, e: - if e.errno != EAGAIN: - raise - - # copy msg lists so they can't be modified by caller - if multi: - msg = list(msg) - - # queue msg to be sent later - self._writers.append((greenlet.getcurrent(), multi, msg, flags, copy, track)) - return self._inner_send_recv() + # The call to send processes 0mq events and may + # make the socket ready to recv. Wake the next + # receiver. (Could check EVENTS for POLLIN here) + if self._recv_event: + self._recv_event.wake() + except __zmq__.ZMQError, e: + if e.errno == EAGAIN: + self._trampoline(True) + else: + raise @_wraps(__zmq__.Socket.recv) def _xsafe_recv(self, flags=0, copy=True, track=False): @@ -272,235 +316,32 @@ class Socket(__zmq__.Socket): self._wake_listener() return msg - return self._xsafe_inner_recv(False, flags, copy, track) - - @_wraps(__zmq__.Socket.recv_multipart) - def _xsafe_recv_multipart(self, flags=0, copy=True, track=False): - """A recv_multipart method that's safe to use when multiple - greenthreads are calling send, send_multipart, recv and - recv_multipart on the same socket. - """ - if flags & __zmq__.NOBLOCK: - msg = super(Socket, self).recv_multipart(flags, copy, track) - self._wake_listener() - return msg - - return self._xsafe_inner_recv(True, flags, copy, track) - - def _xsafe_inner_recv(self, multi, flags, copy, track): flags |= __zmq__.NOBLOCK - if not self._readers: - # no other waiting readers, may be able to recv - # immediately. This is the fast path. - try: - if multi: - msg = super(Socket, self).recv_multipart(flags, copy, track) - else: - msg = super(Socket, self).recv(flags, copy, track) - - self._wake_listener() - return msg - except __zmq__.ZMQError, e: - if e.errno != EAGAIN: - raise - - # queue recv for later - self._readers.append((greenlet.getcurrent(), multi, flags, copy, track)) - return self._inner_send_recv() - - def _inner_send_recv(self): - if self._wake_listener(): - # Another greenthread is listening on the FD. Block this one. - result = hubs.get_hub().switch() - if result is not False: - # msg was sent or received - return result - # Send or recv has not been done, but this thread was - # woken up so that it could process the queues - - return self._process_queues() - - def _process_queues(self): - """If there are readers or writers queued, this method tries - to recv or send messages and ensures processing continues - either in this greenthread or in another one. - """ - readers = self._readers - writers = self._writers - current = greenlet.getcurrent() - - result = None - while True: - try: - # Processing readers before writers here is arbitrary, - # but if you change the order be sure you modify the - # following code that calls getsockopt(EVENTS). - if readers: - result = self._recv_queued() or result - if writers: - result = self._send_queued() or result - except (SystemExit, KeyboardInterrupt): - raise - except: - # an error occurred for this greenthread's send/recv - # call. Wake another thread to continue processing. - if readers: - hubs.get_hub().schedule_call_global(0, readers[0][0].switch, False) - elif writers: - hubs.get_hub().schedule_call_global(0, writers[0][0].switch, False) - raise - - # Above we processed all queued readers and then all - # queued writers. Each call to send or recv can cause the - # zmq to process pending events, so by calling send last - # there may now be a message waiting for a - # reader. However, if we just call recv now then further - # events may notify the socket that a pipe has room for a - # message to be send. To break this vicious cycle and - # safely call trampoline, check getsockopt(EVENTS) to - # ensure a message can't be either sent or received a - # message. - - if readers: - events = self.getsockopt(__zmq__.EVENTS) - if (events & __zmq__.POLLIN) or (writers and (events & __zmq__.POLLOUT)): - # more work to do - continue - - next_reader = readers[0][0] if readers else None - next_writer = writers[0][0] if writers else None - - next_thread = next_reader or next_writer - - # send and recv cannot continue right now. If there are - # more readers or writers queued, either trampoline or - # wake another greenthread. - if next_thread: - # Only trampoline if this thread is the next reader or writer - if next_reader is current or next_writer is current: - self._trampoline() - continue - else: - # This greenthread's work is done. Wake another to - # continue processing the queues if there is one - # blocked. This arbitrarily prefers to wake the - # next reader, but I don't think it matters which. - hubs.get_hub().schedule_call_global(0, next_thread.switch, False) - return result - - def _send_queued(self): - """Send as many msgs from the writers deque as possible. Wake - up the greenthreads for messages that are sent. Continue - sending messages even after this greenthread has sent its own - message because that seems like the best way to increase - throughput but that is an untested assumption. - """ - writers = self._writers - current = greenlet.getcurrent() - hub = hubs.get_hub() - super_send = super(Socket, self).send - super_send_multipart = super(Socket, self).send_multipart - - result = None - - while writers: - writer, multi, msg, flags, copy, track = writers[0] - try: - if multi: - r = super_send_multipart(msg, flags, copy, track) - else: - r = super_send(msg, flags, copy, track) - - # remember this thread's result - if current is writer: - result = r - except (SystemExit, KeyboardInterrupt): - raise - except __zmq__.ZMQError, e: - if e.errno == EAGAIN: - return result - else: - writers.popleft() - if current is writer: - raise + with self._recv_lock: + while True: + try: + try: + return super(Socket, self).recv(flags, copy, track) + finally: + # The call to recv processes 0mq events and may + # make the socket ready to send. Wake the next + # receiver. (Could check EVENTS for POLLOUT here) + if self._send_event: + self._send_event.wake() + except __zmq__.ZMQError, e: + if e.errno == EAGAIN: + self._trampoline(False) else: - hub.schedule_call_global(0, writer.throw, e) - continue - except: - writers.popleft() - if current is writer: - raise - else: - hub.schedule_call_global(0, writer.throw, e) - continue - - # move to the next msg - writers.popleft() - # wake writer - if current is not writer: - hub.schedule_call_global(0, writer.switch, r) - return result - - def _recv_queued(self): - """Recv as many msgs for each of the greenthreads in the - readers deque. Wakes up the greenthreads for messages that are - received. If the received message is for the current - greenthread, returns immediately. - """ - readers = self._readers - super_recv = super(Socket, self).recv - super_recv_multipart = super(Socket, self).recv_multipart - - current = greenlet.getcurrent() - hub = hubs.get_hub() - - while readers: - reader, multi, flags, copy, track = readers[0] - try: - if multi: - msg = super_recv_multipart(flags, copy, track) - else: - msg = super_recv(flags, copy, track) - - except (SystemExit, KeyboardInterrupt): - raise - except __zmq__.ZMQError, e: - if e.errno == EAGAIN: - return None - else: - readers.popleft() - if current is reader: raise - else: - hub.schedule_call_global(0, reader.throw, e) - continue - except: - readers.popleft() - if current is reader: - raise - else: - hub.schedule_call_global(0, reader.throw, e) - continue - - # move to the next reader - readers.popleft() - - if current is reader: - return msg - else: - hub.schedule_call_global(0, reader.switch, msg) - - return None - # The behavior of the send and recv methods depends on the socket # type. See http://api.zeromq.org/2-1:zmq-socket for explanation # of socket types. For the green Socket, our main concern is # supporting calling send or recv from multiple greenthreads when # it makes sense for the socket type. - _send_only_ops = (_xsafe_send, _xsafe_send_multipart, None, None) - _recv_only_ops = (None, None, _xsafe_recv, _xsafe_recv_multipart) - _full_ops = (_xsafe_send, _xsafe_send_multipart, _xsafe_recv, _xsafe_recv_multipart) + _send_only_ops = (_xsafe_send, None) + _recv_only_ops = (None, _xsafe_recv) + _full_ops = (_xsafe_send, _xsafe_recv) _eventlet_ops = { __zmq__.PUB: _send_only_ops, From e71bfc5f8097bbee3b630e676fb90f1773e9f9dc Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Wed, 14 Sep 2011 17:03:33 -0400 Subject: [PATCH 17/41] readd zmq send_multipart and recv_multipart methods to acquire locks --- eventlet/green/zmq.py | 65 ++++++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index e59e540..abd1ffb 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -173,18 +173,20 @@ class Socket(__zmq__.Socket): if ops: self._send_lock = None self._recv_lock = None - send, recv = ops + send, msend, recv, mrecv = ops if send: self._send_lock = _QueueLock() self.send = MethodType(send, self, Socket) + self.send_multipart = MethodType(msend, self, Socket) else: - self.send = self._send_not_supported + self.send = self.send_multipart = self._send_not_supported if recv: self._recv_lock = _QueueLock() self.recv = MethodType(recv, self, Socket) + self.recv_multipart = MethodType(mrecv, self, Socket) else: - self.recv = self._send_not_supported + self.recv = self.recv_multipart = self._send_not_supported def _trampoline(self, is_send): """Wait for events on the zmq socket. After this method @@ -280,7 +282,8 @@ class Socket(__zmq__.Socket): """ if flags & __zmq__.NOBLOCK: result = super(Socket, self).send(msg, flags, copy, track) - self._wake_listener() + if self._send_event or self._recv_event: + getsockopt(__zmq__.EVENTS) # triggers wakeups return result # TODO: pyzmq will copy the message buffer and create Message @@ -290,20 +293,33 @@ class Socket(__zmq__.Socket): with self._send_lock: while True: try: - try: - return super(Socket, self).send(msg, flags, copy, track) - finally: - - # The call to send processes 0mq events and may - # make the socket ready to recv. Wake the next - # receiver. (Could check EVENTS for POLLIN here) - if self._recv_event: - self._recv_event.wake() + return super(Socket, self).send(msg, flags, copy, track) except __zmq__.ZMQError, e: if e.errno == EAGAIN: self._trampoline(True) else: raise + finally: + # The call to send processes 0mq events and may + # make the socket ready to recv. Wake the next + # receiver. (Could check EVENTS for POLLIN here) + if self._recv_event: + self._recv_event.wake() + + + @_wraps(__zmq__.Socket.send_multipart) + def _xsafe_send_multipart(self, msg_parts, flags=0, copy=True, track=False): + """A send_multipart method that's safe to use when multiple + greenthreads are calling send, send_multipart, recv and + recv_multipart on the same socket. + """ + if flags & __zmq__.NOBLOCK: + return super(Socket, self).send_multipart(msg_parts, flags, copy, track) + + # acquire lock here so the subsequent calls to send for the + # message parts after the first don't block + with self._send_lock: + return super(Socket, self).send_multipart(msg_parts, flags, copy, track) @_wraps(__zmq__.Socket.recv) def _xsafe_recv(self, flags=0, copy=True, track=False): @@ -313,7 +329,8 @@ class Socket(__zmq__.Socket): """ if flags & __zmq__.NOBLOCK: msg = super(Socket, self).recv(flags, copy, track) - self._wake_listener() + if self._send_event or self._recv_event: + getsockopt(__zmq__.EVENTS) # triggers wakeups return msg flags |= __zmq__.NOBLOCK @@ -334,14 +351,28 @@ class Socket(__zmq__.Socket): else: raise + @_wraps(__zmq__.Socket.recv_multipart) + def _xsafe_recv_multipart(self, flags=0, copy=True, track=False): + """A recv_multipart method that's safe to use when multiple + greenthreads are calling send, send_multipart, recv and + recv_multipart on the same socket. + """ + if flags & __zmq__.NOBLOCK: + return super(Socket, self).recv_multipart(flags, copy, track) + + # acquire lock here so the subsequent calls to recv for the + # message parts after the first don't block + with self._recv_lock: + return super(Socket, self).recv_multipart(flags, copy, track) + # The behavior of the send and recv methods depends on the socket # type. See http://api.zeromq.org/2-1:zmq-socket for explanation # of socket types. For the green Socket, our main concern is # supporting calling send or recv from multiple greenthreads when # it makes sense for the socket type. - _send_only_ops = (_xsafe_send, None) - _recv_only_ops = (None, _xsafe_recv) - _full_ops = (_xsafe_send, _xsafe_recv) + _send_only_ops = (_xsafe_send, _xsafe_send_multipart, None, None) + _recv_only_ops = (None, None, _xsafe_recv, _xsafe_recv_multipart) + _full_ops = (_xsafe_send, _xsafe_send_multipart, _xsafe_recv, _xsafe_recv_multipart) _eventlet_ops = { __zmq__.PUB: _send_only_ops, From 8d69bc5d92ce3d1a215818f2d6e91f37d95509d6 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Wed, 14 Sep 2011 22:31:59 -0400 Subject: [PATCH 18/41] add tests for zmq._SimpleEvent and zmq._QueueLock --- eventlet/green/zmq.py | 6 +- tests/zmq_test.py | 131 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 3 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index abd1ffb..d34191a 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -47,7 +47,8 @@ class _QueueLock(object): self._count += 1 def release(self): - assert self._count > 0, 'Cannot release unacquired lock' + if self._count <= 0: + raise Exception("Cannot release unacquired lock") self._count -= 1 if self._count == 0: @@ -75,7 +76,8 @@ class _SimpleEvent(object): hubs.get_hub().switch() def __enter__(self): - assert self._blocked_thread is None, 'Cannot block more than one thread on one SimpleEvent' + if self._blocked_thread is not None: + raise Exception("Cannot block more than one thread on one SimpleEvent") self._blocked_thread = greenlet.getcurrent() def __exit__(self, type, value, traceback): diff --git a/tests/zmq_test.py b/tests/zmq_test.py index c3392e4..79247df 100644 --- a/tests/zmq_test.py +++ b/tests/zmq_test.py @@ -1,4 +1,4 @@ -from eventlet import event, spawn, sleep, patcher +from eventlet import event, spawn, sleep, patcher, semaphore from eventlet.hubs import get_hub, _threadlocal, use_hub from nose.tools import * from tests import mock, LimitedTestCase, using_pyevent, skip_unless @@ -342,3 +342,132 @@ class TestThreadedContextAccess(TestCase): sleep(0.1) self.assertFalse(test_result[0]) +class TestQueueLock(LimitedTestCase): + @skip_unless(zmq_supported) + def test_queue_lock_order(self): + q = zmq._QueueLock() + s = semaphore.Semaphore(0) + results = [] + + def lock(x): + with q: + results.append(x) + s.release() + + q.acquire() + + spawn(lock, 1) + sleep() + spawn(lock, 2) + sleep() + spawn(lock, 3) + sleep() + + self.assertEquals(results, []) + q.release() + s.acquire() + s.acquire() + s.acquire() + self.assertEquals(results, [1,2,3]) + + @skip_unless(zmq_supported) + def test_count(self): + q = zmq._QueueLock() + self.assertFalse(q) + q.acquire() + self.assertTrue(q) + q.release() + self.assertFalse(q) + + with q: + self.assertTrue(q) + self.assertFalse(q) + + @skip_unless(zmq_supported) + def test_errors(self): + q = zmq._QueueLock() + + with self.assertRaises(Exception): + q.release() + + q.acquire() + q.release() + + with self.assertRaises(Exception): + q.release() + + @skip_unless(zmq_supported) + def test_nested_acquire(self): + q = zmq._QueueLock() + self.assertFalse(q) + q.acquire() + q.acquire() + + s = semaphore.Semaphore(0) + results = [] + def lock(x): + with q: + results.append(x) + s.release() + + spawn(lock, 1) + sleep() + self.assertEquals(results, []) + q.release() + sleep() + self.assertEquals(results, []) + self.assertTrue(q) + q.release() + + s.acquire() + self.assertEquals(results, [1]) + +class TestSimpleEvent(LimitedTestCase): + @skip_unless(zmq_supported) + def test_block(self): + e = zmq._SimpleEvent() + done = event.Event() + self.assertFalse(e) + + def block(): + e.block() + done.send(1) + + spawn(block) + sleep() + + self.assertFalse(done.has_result()) + e.wake() + done.wait() + + @skip_unless(zmq_supported) + def test_enter_exit(self): + e = zmq._SimpleEvent() + done = event.Event() + self.assertFalse(e) + + def block(): + with e: + get_hub().switch() + done.send(1) + + gt = spawn(block) + sleep() + + self.assertFalse(done.has_result()) + get_hub().schedule_call_global(0, gt.switch) + done.wait() + + + @skip_unless(zmq_supported) + def test_error(self): + e1 = zmq._SimpleEvent() + with e1: + with self.assertRaises(Exception): + with e1: + pass + + e2 = zmq._SimpleEvent() + with e2: + with self.assertRaises(Exception): + e2.block() From e3b9eb1f5b7953b33c0746e3d637231a1d6b497b Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Wed, 14 Sep 2011 22:36:34 -0400 Subject: [PATCH 19/41] Not using zmq_poll anymore, so we can allow multiple zmq.Context objects in the same native thread. --- eventlet/green/zmq.py | 24 ++---------------------- tests/zmq_test.py | 42 ------------------------------------------ 2 files changed, 2 insertions(+), 64 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index d34191a..bfb26c4 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -99,28 +99,8 @@ class _SimpleEvent(object): return True return False -def Context(io_threads=1): - """Factory function replacement for :class:`zmq.core.context.Context` - - This factory ensures the :class:`zeromq hub ` - is the active hub, and defers creation (or retreival) of the ``Context`` - to the hub's :meth:`~eventlet.hubs.zeromq.Hub.get_context` method - - It's a factory function due to the fact that there can only be one :class:`_Context` - instance per thread. This is due to the way :class:`zmq.core.poll.Poller` - works - """ - try: - return _threadlocal.context - except AttributeError: - _threadlocal.context = _Context(io_threads) - return _threadlocal.context - -class _Context(__zmq__.Context): - """Internal subclass of :class:`zmq.core.context.Context` - - .. warning:: Do not grab one of these yourself, use the factory function - :func:`eventlet.green.zmq.Context` +class Context(__zmq__.Context): + """Subclass of :class:`zmq.core.context.Context` """ def socket(self, socket_type): diff --git a/tests/zmq_test.py b/tests/zmq_test.py index 79247df..8be1ab9 100644 --- a/tests/zmq_test.py +++ b/tests/zmq_test.py @@ -300,48 +300,6 @@ got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) final_i = done.wait() self.assertEqual(final_i, 0) - -class TestThreadedContextAccess(TestCase): - """zmq's Context must be unique within a hub - - The zeromq API documentation states: - All zmq sockets passed to the zmq_poll() function must share the same zmq - context and must belong to the thread calling zmq_poll() - - As zmq_poll is what's eventually being called then we need to ensure that - all sockets that are going to be passed to zmq_poll (via hub.do_poll) are - in the same context - """ - if zmq: # don't call decorators if zmq module unavailable - @skip_unless(zmq_supported) - def test_context_factory_function(self): - ctx = zmq.Context() - self.assertTrue(ctx is not None) - - @skip_unless(zmq_supported) - def test_threadlocal_context(self): - context = zmq.Context() - self.assertEqual(context, _threadlocal.context) - next_context = zmq.Context() - self.assertTrue(context is next_context) - - @skip_unless(zmq_supported) - def test_different_context_in_different_thread(self): - context = zmq.Context() - test_result = [] - def assert_different(ctx): - try: - this_thread_context = zmq.Context() - except: - test_result.append('fail') - raise - test_result.append(ctx is this_thread_context) - - Thread(target=assert_different, args=(context,)).start() - while not test_result: - sleep(0.1) - self.assertFalse(test_result[0]) - class TestQueueLock(LimitedTestCase): @skip_unless(zmq_supported) def test_queue_lock_order(self): From 33aa6ee5cb8fe227f5a0695c8001de39bdddf10c Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Wed, 14 Sep 2011 22:44:10 -0400 Subject: [PATCH 20/41] Add test for multipart zmq messages --- tests/zmq_test.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/zmq_test.py b/tests/zmq_test.py index 8be1ab9..f8af782 100644 --- a/tests/zmq_test.py +++ b/tests/zmq_test.py @@ -274,6 +274,43 @@ got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) final_i = done_evts[i].wait() self.assertEqual(final_i, 0) + + @skip_unless(zmq_supported) + def test_send_during_recv_multipart(self): + sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) + sleep() + + num_recvs = 1 + done_evts = [event.Event() for _ in range(num_recvs)] + + def slow_rx(done, msg): + self.assertEqual(sender.recv_multipart(), msg) + done.send(0) + + def tx(): + tx_i = 0 + while tx_i <= 1000: + sender.send_multipart([str(tx_i), '1', '2', '3']) + tx_i += 1 + + def rx(): + while True: + rx_i = receiver.recv_multipart() + if rx_i == ["1000", '1', '2', '3']: + for i in range(num_recvs): + receiver.send_multipart(['done%d' % i, 'a', 'b', 'c']) + sleep() + return + + for i in range(num_recvs): + spawn(slow_rx, done_evts[i], ["done%d" % i, 'a', 'b', 'c']) + + spawn(tx) + spawn(rx) + for i in range(num_recvs): + final_i = done_evts[i].wait() + self.assertEqual(final_i, 0) + # Need someway to ensure a thread is blocked on send... This isn't working @skip_unless(zmq_supported) From a6188780666c0c122a0c1e9299de13edb7b5aa55 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Wed, 14 Sep 2011 22:46:11 -0400 Subject: [PATCH 21/41] Tweak multipart message zmq test --- tests/zmq_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/zmq_test.py b/tests/zmq_test.py index f8af782..3f20ba5 100644 --- a/tests/zmq_test.py +++ b/tests/zmq_test.py @@ -280,7 +280,7 @@ got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) sleep() - num_recvs = 1 + num_recvs = 30 done_evts = [event.Event() for _ in range(num_recvs)] def slow_rx(done, msg): From 389193c8414bc1e88c004fe5d3819a82e5539a1b Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 15 Sep 2011 15:30:23 -0400 Subject: [PATCH 22/41] adds comments explaining zmq greening --- eventlet/green/zmq.py | 75 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index bfb26c4..896e7cd 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -112,15 +112,6 @@ class Context(__zmq__.Context): """ return Socket(self, socket_type) - -# TODO: -# - Ensure that recv* and send* methods raise error when called on a -# closed socket. They should not block. -# - Return correct message tracker from send* methods -# - Make MessageTracker.wait zmq friendly -# - What should happen to threads blocked on send/recv when socket is -# closed? - def _wraps(source_fn): """A decorator that copies the __name__ and __doc__ from the given function @@ -131,16 +122,80 @@ def _wraps(source_fn): return dest_fn return wrapper +# Implementation notes: Each socket in 0mq contains a pipe that the +# background IO threads use to communicate with the socket. These +# events are important because they tell the socket when it is able to +# send and when it has messages waiting to be received. The read end +# of the events pipe is the same FD that getsockopt(zmq.FD) returns. +# +# Events are read from the socket's event pipe only on the thread that +# the 0mq context is associated with, which is the native thread the +# greenthreads are running on, and the only operations that cause the +# events to be read and processed are send(), recv() and +# getsockopt(EVENTS). This means that after doing any of these three +# operations, the ability of the socket to send or receive a message +# without blocking may have changed. If you're not careful, this can +# cause the hub to miss the read event for the socket. +# +# For example, suppose thread A calls trampoline and blocks because it +# called recv() when there was no waiting message. It should be +# notified when the state of the socket changes. However, while thread +# A is blocked, thread B calls send(), which internally causes the +# events to be processed, and the socket learns that it has a message +# waiting to be received. Unfortunately, because eventlet is currently +# running greenthread B, it isn't currently blocked in hub.wait() in +# poll or the equivalent. When hub.wait() is eventually called, the +# socket's event pipe will no longer be readable, so thread A will not +# be awoken, even though a message is waiting to be read! +# +# If we understand that after calling send() a message might be ready +# to be received and that after calling recv() a message might be able +# to be sent, what should we do next? There are two approaches: +# +# 1. Always wake the other thread if there is one waiting. This +# wakeup may be spurious because the socket might not actually be +# ready for a send() or recv(). However, if a thread is in a +# tight-loop successfully calling send() or recv() then the wakeups +# are naturally batched and there's very little cost added to each +# send/recv call. +# +# or +# +# 2. Call getsockopt(zmq.EVENTS) and explicitly check if the other +# thread should be woken up. This avoids spurious wake-ups but may +# add overhead because getsockopt will cause all events to be +# processed, whereas send and recv can avoid processing +# events. Admittedly, all of the events will need to be processed +# eventually, but it is likely faster to batch the processing. +# +# Which approach is better? I have no idea. Right now the NOBLOCK +# paths in _xsafe_send and _xsafe_recv check getsockopt(zmq.EVENTS) +# and the other paths always wake the other blocked thread. It's done +# this way only because it was convenient to implement, not based on +# any benchmarks. +# +# TODO: +# - Ensure that recv* and send* methods raise error when called on a +# closed socket. They should not block. +# - Return correct message tracker from send* methods +# - Make MessageTracker.wait zmq friendly +# - What should happen to threads blocked on send/recv when socket is +# closed? + class Socket(__zmq__.Socket): """Green version of :class:`zmq.core.socket.Socket - The following two methods are always overridden: + The following three methods are always overridden: * send * recv * getsockopt To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or recieving is deferred to the hub (using :func:`eventlet.hubs.trampoline`) if a ``zmq.EAGAIN`` (retry) error is raised + + For some socket types, the following methods are also overridden: + * send_multipart + * recv_multipart """ def __init__(self, context, socket_type): From 38c420167493b49f676230f90871fdfb75d3bc8f Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Fri, 16 Sep 2011 11:14:24 -0400 Subject: [PATCH 23/41] add closed check to Context.socket(). pyzmq's socket() contains the same check. --- eventlet/green/zmq.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 896e7cd..6634277 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -110,6 +110,8 @@ class Context(__zmq__.Context): that a :class:`Socket` with all of its send and recv methods set to be non-blocking is returned """ + if self.closed: + raise ZMQError(ENOTSUP) return Socket(self, socket_type) def _wraps(source_fn): From 0f8ca258b727a473ea1d48e812e702c25e5a05c1 Mon Sep 17 00:00:00 2001 From: Sergey Shepelev Date: Wed, 12 Oct 2011 18:37:38 +0300 Subject: [PATCH 24/41] Fix db_pool .clear() when min_size > 0 --- eventlet/db_pool.py | 4 +++- tests/db_pool_test.py | 14 ++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/eventlet/db_pool.py b/eventlet/db_pool.py index 9874c2f..2664a95 100644 --- a/eventlet/db_pool.py +++ b/eventlet/db_pool.py @@ -228,7 +228,9 @@ class BaseConnectionPool(Pool): if self._expiration_timer: self._expiration_timer.cancel() free_items, self.free_items = self.free_items, deque() - for _last_used, _created_at, conn in free_items: + for item in free_items: + # Free items created using min_size>0 are not tuples. + conn = item[2] if isinstance(item, tuple) else item self._safe_close(conn, quiet=True) def __del__(self): diff --git a/tests/db_pool_test.py b/tests/db_pool_test.py index ed6f09b..2a316b0 100644 --- a/tests/db_pool_test.py +++ b/tests/db_pool_test.py @@ -247,6 +247,12 @@ class DBConnectionPool(DBTester): self.pool.clear() self.assertEqual(len(self.pool.free_items), 0) + def test_clear_warmup(self): + """Clear implicitly created connections (min_size > 0)""" + self.pool = self.create_pool(min_size=1) + self.pool.clear() + self.assertEqual(len(self.pool.free_items), 0) + def test_unwrap_connection(self): self.assert_(isinstance(self.connection, db_pool.GenericConnectionWrapper)) @@ -438,12 +444,12 @@ class RaisingDBModule(object): class TpoolConnectionPool(DBConnectionPool): __test__ = False # so that nose doesn't try to execute this directly - def create_pool(self, max_size=1, max_idle=10, max_age=10, + def create_pool(self, min_size=0, max_size=1, max_idle=10, max_age=10, connect_timeout=0.5, module=None): if module is None: module = self._dbmodule return db_pool.TpooledConnectionPool(module, - min_size=0, max_size=max_size, + min_size=min_size, max_size=max_size, max_idle=max_idle, max_age=max_age, connect_timeout = connect_timeout, **self._auth) @@ -462,12 +468,12 @@ class TpoolConnectionPool(DBConnectionPool): class RawConnectionPool(DBConnectionPool): __test__ = False # so that nose doesn't try to execute this directly - def create_pool(self, max_size=1, max_idle=10, max_age=10, + def create_pool(self, min_size=0, max_size=1, max_idle=10, max_age=10, connect_timeout=0.5, module=None): if module is None: module = self._dbmodule return db_pool.RawConnectionPool(module, - min_size=0, max_size=max_size, + min_size=min_size, max_size=max_size, max_idle=max_idle, max_age=max_age, connect_timeout=connect_timeout, **self._auth) From e889c85dae4164de7f6414fe76dc55c4102cf30a Mon Sep 17 00:00:00 2001 From: Gregory Holt Date: Thu, 20 Oct 2011 03:11:57 +0000 Subject: [PATCH 25/41] Limit HTTP header line length --- eventlet/wsgi.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/eventlet/wsgi.py b/eventlet/wsgi.py index e9959e1..15fcba3 100644 --- a/eventlet/wsgi.py +++ b/eventlet/wsgi.py @@ -15,6 +15,7 @@ from eventlet.support import get_errno DEFAULT_MAX_SIMULTANEOUS_REQUESTS = 1024 DEFAULT_MAX_HTTP_VERSION = 'HTTP/1.1' MAX_REQUEST_LINE = 8192 +MAX_HEADER_LINE = 8192 MINIMUM_CHUNK_SIZE = 4096 DEFAULT_LOG_FORMAT= ('%(client_ip)s - - [%(date_time)s] "%(request_line)s"' ' %(status_code)s %(body_length)s %(wall_seconds).6f') @@ -163,6 +164,25 @@ class Input(object): return self.rfile._sock +class ReadlineTooLong(Exception): + pass + + +class FileObjectForHeaders(object): + + def __init__(self, fp): + self.fp = fp + + def readline(self, size=-1): + sz = size + if size < 0: + sz = MAX_HEADER_LINE + rv = self.fp.readline(sz) + if size < 0 and len(rv) == MAX_HEADER_LINE: + raise ReadlineTooLong() + return rv + + class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler): protocol_version = 'HTTP/1.1' minimum_chunk_size = MINIMUM_CHUNK_SIZE @@ -210,8 +230,19 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler): self.close_connection = 1 return - if not self.parse_request(): + orig_rfile = self.rfile + try: + self.rfile = FileObjectForHeaders(self.rfile) + if not self.parse_request(): + return + except ReadlineTooLong: + self.wfile.write( + "HTTP/1.0 400 Header Line Too Long\r\n" + "Connection: close\r\nContent-length: 0\r\n\r\n") + self.close_connection = 1 return + finally: + self.rfile = orig_rfile content_length = self.headers.getheader('content-length') if content_length: From fa59a3b339e103bd2660c3cf2204bbd1da297ba3 Mon Sep 17 00:00:00 2001 From: Gregory Holt Date: Thu, 20 Oct 2011 14:37:22 +0000 Subject: [PATCH 26/41] Limit overall HTTP header length and tests --- eventlet/wsgi.py | 23 +++++++++++++++++++---- tests/wsgi_test.py | 24 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/eventlet/wsgi.py b/eventlet/wsgi.py index 15fcba3..6003664 100644 --- a/eventlet/wsgi.py +++ b/eventlet/wsgi.py @@ -16,6 +16,7 @@ DEFAULT_MAX_SIMULTANEOUS_REQUESTS = 1024 DEFAULT_MAX_HTTP_VERSION = 'HTTP/1.1' MAX_REQUEST_LINE = 8192 MAX_HEADER_LINE = 8192 +MAX_TOTAL_HEADER_SIZE = 65536 MINIMUM_CHUNK_SIZE = 4096 DEFAULT_LOG_FORMAT= ('%(client_ip)s - - [%(date_time)s] "%(request_line)s"' ' %(status_code)s %(body_length)s %(wall_seconds).6f') @@ -164,7 +165,11 @@ class Input(object): return self.rfile._sock -class ReadlineTooLong(Exception): +class HeaderLineTooLong(Exception): + pass + + +class HeadersTooLarge(Exception): pass @@ -172,14 +177,18 @@ class FileObjectForHeaders(object): def __init__(self, fp): self.fp = fp + self.total_header_size = 0 def readline(self, size=-1): sz = size if size < 0: sz = MAX_HEADER_LINE rv = self.fp.readline(sz) - if size < 0 and len(rv) == MAX_HEADER_LINE: - raise ReadlineTooLong() + if size < 0 and len(rv) >= MAX_HEADER_LINE: + raise HeaderLineTooLong() + self.total_header_size += len(rv) + if self.total_header_size > MAX_TOTAL_HEADER_SIZE: + raise HeadersTooLarge() return rv @@ -235,12 +244,18 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler): self.rfile = FileObjectForHeaders(self.rfile) if not self.parse_request(): return - except ReadlineTooLong: + except HeaderLineTooLong: self.wfile.write( "HTTP/1.0 400 Header Line Too Long\r\n" "Connection: close\r\nContent-length: 0\r\n\r\n") self.close_connection = 1 return + except HeadersTooLarge: + self.wfile.write( + "HTTP/1.0 400 Headers Too Large\r\n" + "Connection: close\r\nContent-length: 0\r\n\r\n") + self.close_connection = 1 + return finally: self.rfile = orig_rfile diff --git a/tests/wsgi_test.py b/tests/wsgi_test.py index b257efb..ecdb688 100644 --- a/tests/wsgi_test.py +++ b/tests/wsgi_test.py @@ -871,6 +871,30 @@ class TestHttpd(_TestBase): self.assertEquals(posthook1_count[0], 26) self.assertEquals(posthook2_count[0], 25) + def test_030_reject_long_header_lines(self): + sock = eventlet.connect(('localhost', self.port)) + request = 'GET / HTTP/1.0\r\nHost: localhost\r\nLong: %s\r\n\r\n' % \ + ('a' * 10000) + fd = sock.makefile('rw') + fd.write(request) + fd.flush() + response_line, headers, body = read_http(sock) + self.assertEquals(response_line, + 'HTTP/1.0 400 Header Line Too Long\r\n') + fd.close() + + def test_031_reject_large_headers(self): + sock = eventlet.connect(('localhost', self.port)) + headers = 'Name: Value\r\n' * 5050 + request = 'GET / HTTP/1.0\r\nHost: localhost\r\n%s\r\n\r\n' % headers + fd = sock.makefile('rw') + fd.write(request) + fd.flush() + response_line, headers, body = read_http(sock) + self.assertEquals(response_line, + 'HTTP/1.0 400 Headers Too Large\r\n') + fd.close() + def test_zero_length_chunked_response(self): def zero_chunked_app(env, start_response): start_response('200 OK', [('Content-type', 'text/plain')]) From f1bed7d9f95cbe4d44e1dd0a22645002d6183869 Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Fri, 28 Oct 2011 23:12:12 -0700 Subject: [PATCH 27/41] Added a missing traceback import. --- tests/mysqldb_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/mysqldb_test.py b/tests/mysqldb_test.py index 7eb3e52..f6f45cf 100644 --- a/tests/mysqldb_test.py +++ b/tests/mysqldb_test.py @@ -1,6 +1,7 @@ import os import sys import time +import traceback from tests import skipped, skip_unless, using_pyevent, get_database_auth, LimitedTestCase import eventlet from eventlet import event From 21f905dbc6f7acccffa9e74711e1a57885e0e9c9 Mon Sep 17 00:00:00 2001 From: "jmg.utn" Date: Sun, 6 Nov 2011 14:33:56 -0300 Subject: [PATCH 28/41] log_output flag parameter added to the wsgi server --- eventlet/wsgi.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/eventlet/wsgi.py b/eventlet/wsgi.py index e9959e1..0ae5389 100644 --- a/eventlet/wsgi.py +++ b/eventlet/wsgi.py @@ -383,14 +383,16 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler): for hook, args, kwargs in self.environ['eventlet.posthooks']: hook(self.environ, *args, **kwargs) - - self.server.log_message(self.server.log_format % dict( - client_ip=self.get_client_ip(), - date_time=self.log_date_time_string(), - request_line=self.requestline, - status_code=status_code[0], - body_length=length[0], - wall_seconds=finish - start)) + + if self.server.log_output: + + self.server.log_message(self.server.log_format % dict( + client_ip=self.get_client_ip(), + date_time=self.log_date_time_string(), + request_line=self.requestline, + status_code=status_code[0], + body_length=length[0], + wall_seconds=finish - start)) def get_client_ip(self): client_ip = self.client_address[0] @@ -471,6 +473,7 @@ class Server(BaseHTTPServer.HTTPServer): minimum_chunk_size=None, log_x_forwarded_for=True, keepalive=True, + log_output=True, log_format=DEFAULT_LOG_FORMAT, debug=True): @@ -490,6 +493,7 @@ class Server(BaseHTTPServer.HTTPServer): if minimum_chunk_size is not None: protocol.minimum_chunk_size = minimum_chunk_size self.log_x_forwarded_for = log_x_forwarded_for + self.log_output = log_output self.log_format = log_format self.debug = debug @@ -536,7 +540,8 @@ def server(sock, site, minimum_chunk_size=None, log_x_forwarded_for=True, custom_pool=None, - keepalive=True, + keepalive=True, + log_output=True, log_format=DEFAULT_LOG_FORMAT, debug=True): """ Start up a wsgi server handling requests from the supplied server @@ -556,6 +561,7 @@ def server(sock, site, :param log_x_forwarded_for: If True (the default), logs the contents of the x-forwarded-for header in addition to the actual client ip address in the 'client_ip' field of the log line. :param custom_pool: A custom GreenPool instance which is used to spawn client green threads. If this is supplied, max_size is ignored. :param keepalive: If set to False, disables keepalives on the server; all connections will be closed after serving one request. + :param log_output: A Boolean indicating if the server will log data or not. :param log_format: A python format string that is used as the template to generate log lines. The following values can be formatted into it: client_ip, date_time, request_line, status_code, body_length, wall_seconds. The default is a good example of how to use it. :param debug: True if the server should send exception tracebacks to the clients on 500 errors. If False, the server will respond with empty bodies. """ @@ -567,6 +573,7 @@ def server(sock, site, minimum_chunk_size=minimum_chunk_size, log_x_forwarded_for=log_x_forwarded_for, keepalive=keepalive, + log_output=log_output, log_format=log_format, debug=debug) if server_event is not None: From 4545bf1713b3d84fdf291e33460426b01fea7fa1 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 29 Dec 2011 11:01:43 -0500 Subject: [PATCH 29/41] zmq: cache super() calls. fix calls to getsockopt. remove unnecessary __zmq__ references. --- eventlet/green/zmq.py | 126 ++++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 60 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 6634277..f1ebe95 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -1,8 +1,8 @@ """The :mod:`zmq` module wraps the :class:`Socket` and :class:`Context` found in :mod:`pyzmq ` to be non blocking """ __zmq__ = __import__('zmq') -from eventlet import sleep, hubs -from eventlet.hubs import trampoline, _threadlocal +from eventlet import hubs +from eventlet.hubs import trampoline from eventlet.patcher import slurp_properties from eventlet.support import greenlets as greenlet @@ -184,7 +184,14 @@ def _wraps(source_fn): # - What should happen to threads blocked on send/recv when socket is # closed? -class Socket(__zmq__.Socket): +_Socket = __zmq__.Socket +_Socket_recv = _Socket.recv +_Socket_send = _Socket.send +_Socket_send_multipart = _Socket.send_multipart +_Socket_recv_multipart = _Socket.recv_multipart +_Socket_getsockopt = _Socket.getsockopt + +class Socket(_Socket): """Green version of :class:`zmq.core.socket.Socket The following three methods are always overridden: @@ -227,7 +234,7 @@ class Socket(__zmq__.Socket): else: self.recv = self.recv_multipart = self._send_not_supported - def _trampoline(self, is_send): + def _trampoline(self, evt): """Wait for events on the zmq socket. After this method returns it is still possible that send and recv will return EAGAIN. @@ -238,7 +245,6 @@ class Socket(__zmq__.Socket): still block. """ - evt = self._send_event if is_send else self._recv_event if self._in_trampoline: # Already a thread blocked in trampoline. evt.block() @@ -247,95 +253,95 @@ class Socket(__zmq__.Socket): self._in_trampoline = True with evt: # Only trampoline on read events for zmq FDs, never write. - trampoline(self.getsockopt(__zmq__.FD), read=True) + trampoline(self.getsockopt(FD), read=True) finally: self._in_trampoline = False - @_wraps(__zmq__.Socket.send) + @_wraps(_Socket.send) def send(self, msg, flags=0, copy=True, track=False): """Send method used by REP and REQ sockets. The lock-step send->recv->send->recv restriction of these sockets makes this implementation simple. """ - if flags & __zmq__.NOBLOCK: - return super(Socket, self).send(msg, flags, copy, track) + if flags & NOBLOCK: + return _Socket_send(self, msg, flags, copy, track) - flags |= __zmq__.NOBLOCK + flags |= NOBLOCK while True: try: - return super(Socket, self).send(msg, flags, copy, track) - except __zmq__.ZMQError, e: + return _Socket_send(self, msg, flags, copy, track) + except ZMQError, e: if e.errno == EAGAIN: - self._trampoline(True) + self._trampoline(self._send_event) else: raise - @_wraps(__zmq__.Socket.recv) + @_wraps(_Socket.recv) def recv(self, flags=0, copy=True, track=False): """Recv method used by REP and REQ sockets. The lock-step send->recv->send->recv restriction of these sockets makes this implementation simple. """ - if flags & __zmq__.NOBLOCK: - return super(Socket, self).recv(flags, copy, track) + if flags & NOBLOCK: + return _Socket_recv(self, flags, copy, track) - flags |= __zmq__.NOBLOCK + flags |= NOBLOCK while True: try: - return super(Socket, self).recv(flags, copy, track) - except __zmq__.ZMQError, e: + return _Socket_recv(self, flags, copy, track) + except ZMQError, e: if e.errno == EAGAIN: - self._trampoline(False) + self._trampoline(self._recv_event) else: raise - @_wraps(__zmq__.Socket.getsockopt) + @_wraps(_Socket.getsockopt) def getsockopt(self, option): - result = super(Socket, self).getsockopt(option) - if option == __zmq__.EVENTS: + result = _Socket_getsockopt(self, option) + if option == EVENTS: # Getting the events causes the zmq socket to process # events which may mean a msg can be sent or received. If # there is a greenthread blocked and waiting for events, # it will miss the edge-triggered read event, so wake it # up. - if self._send_evt and (result & __zmq__.POLLOUT): + if self._send_evt and (result & POLLOUT): self._send_evt.wake() - if self._recv_evt and (result & __zmq__.POLLIN): + if self._recv_evt and (result & POLLIN): self._recv_evt.wake() return result def _send_not_supported(self, msg, flags, copy, track): - raise __zmq__.ZMQError(__zmq__.ENOTSUP) + raise ZMQError(ENOTSUP) def _recv_not_supported(self, flags, copy, track): - raise __zmq__.ZMQError(__zmq__.ENOTSUP) + raise ZMQError(ENOTSUP) - @_wraps(__zmq__.Socket.send) + @_wraps(_Socket.send) def _xsafe_send(self, msg, flags=0, copy=True, track=False): """A send method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. """ - if flags & __zmq__.NOBLOCK: - result = super(Socket, self).send(msg, flags, copy, track) + if flags & NOBLOCK: + result = _Socket_send(self, msg, flags, copy, track) if self._send_event or self._recv_event: - getsockopt(__zmq__.EVENTS) # triggers wakeups + self.getsockopt(EVENTS) # triggers wakeups return result # TODO: pyzmq will copy the message buffer and create Message # objects under some circumstances. We could do that work here # once to avoid doing it every time the send is retried. - flags |= __zmq__.NOBLOCK + flags |= NOBLOCK with self._send_lock: while True: try: - return super(Socket, self).send(msg, flags, copy, track) - except __zmq__.ZMQError, e: + return _Socket_send(self, msg, flags, copy, track) + except ZMQError, e: if e.errno == EAGAIN: - self._trampoline(True) + self._trampoline(self._send_event) else: raise finally: @@ -346,63 +352,63 @@ class Socket(__zmq__.Socket): self._recv_event.wake() - @_wraps(__zmq__.Socket.send_multipart) + @_wraps(_Socket.send_multipart) def _xsafe_send_multipart(self, msg_parts, flags=0, copy=True, track=False): """A send_multipart method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. """ - if flags & __zmq__.NOBLOCK: - return super(Socket, self).send_multipart(msg_parts, flags, copy, track) + if flags & NOBLOCK: + return _Socket_send_multipart(self, msg_parts, flags, copy, track) # acquire lock here so the subsequent calls to send for the # message parts after the first don't block with self._send_lock: - return super(Socket, self).send_multipart(msg_parts, flags, copy, track) + return _Socket_send_multipart(self, msg_parts, flags, copy, track) - @_wraps(__zmq__.Socket.recv) + @_wraps(_Socket.recv) def _xsafe_recv(self, flags=0, copy=True, track=False): """A recv method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. """ - if flags & __zmq__.NOBLOCK: - msg = super(Socket, self).recv(flags, copy, track) + if flags & NOBLOCK: + msg = _Socket_recv(self, flags, copy, track) if self._send_event or self._recv_event: - getsockopt(__zmq__.EVENTS) # triggers wakeups + self.getsockopt(EVENTS) # triggers wakeups return msg - flags |= __zmq__.NOBLOCK + flags |= NOBLOCK with self._recv_lock: while True: try: try: - return super(Socket, self).recv(flags, copy, track) + return _Socket_recv(self, flags, copy, track) finally: # The call to recv processes 0mq events and may # make the socket ready to send. Wake the next # receiver. (Could check EVENTS for POLLOUT here) if self._send_event: self._send_event.wake() - except __zmq__.ZMQError, e: + except ZMQError, e: if e.errno == EAGAIN: - self._trampoline(False) + self._trampoline(self._recv_event) else: raise - @_wraps(__zmq__.Socket.recv_multipart) + @_wraps(_Socket.recv_multipart) def _xsafe_recv_multipart(self, flags=0, copy=True, track=False): """A recv_multipart method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. """ - if flags & __zmq__.NOBLOCK: - return super(Socket, self).recv_multipart(flags, copy, track) + if flags & NOBLOCK: + return _Socket_recv_multipart(self, flags, copy, track) # acquire lock here so the subsequent calls to recv for the # message parts after the first don't block with self._recv_lock: - return super(Socket, self).recv_multipart(flags, copy, track) + return _Socket_recv_multipart(self, flags, copy, track) # The behavior of the send and recv methods depends on the socket # type. See http://api.zeromq.org/2-1:zmq-socket for explanation @@ -414,19 +420,19 @@ class Socket(__zmq__.Socket): _full_ops = (_xsafe_send, _xsafe_send_multipart, _xsafe_recv, _xsafe_recv_multipart) _eventlet_ops = { - __zmq__.PUB: _send_only_ops, - __zmq__.SUB: _recv_only_ops, + PUB: _send_only_ops, + SUB: _recv_only_ops, - __zmq__.PUSH: _send_only_ops, - __zmq__.PULL: _recv_only_ops, + PUSH: _send_only_ops, + PULL: _recv_only_ops, - __zmq__.PAIR: _full_ops + PAIR: _full_ops } try: - _eventlet_ops[__zmq__.XREP] = _full_ops - _eventlet_ops[__zmq__.XREQ] = _full_ops + _eventlet_ops[XREP] = _full_ops + _eventlet_ops[XREQ] = _full_ops except AttributeError: # XREP and XREQ are being renamed ROUTER and DEALER - _eventlet_ops[__zmq__.ROUTER] = _full_ops - _eventlet_ops[__zmq__.DEALER] = _full_ops + _eventlet_ops[ROUTER] = _full_ops + _eventlet_ops[DEALER] = _full_ops From 301467e18345f3eff742ba698e7722530f33f32e Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 26 Jan 2012 13:50:08 -0500 Subject: [PATCH 30/41] don't change version for now --- eventlet/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eventlet/__init__.py b/eventlet/__init__.py index 2e47faf..9d92c75 100644 --- a/eventlet/__init__.py +++ b/eventlet/__init__.py @@ -1,4 +1,4 @@ -version_info = (0, 9, 17, "dev") +version_info = (0, 9, 16, 1) __version__ = ".".join(map(str, version_info)) try: From 7db155ab7493315a5a8e3858413c41f3acd95b09 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 26 Jan 2012 13:52:39 -0500 Subject: [PATCH 31/41] simplify test --- tests/zmq_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/zmq_test.py b/tests/zmq_test.py index 3f20ba5..67f9561 100644 --- a/tests/zmq_test.py +++ b/tests/zmq_test.py @@ -270,9 +270,8 @@ got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) spawn(tx) spawn(rx) - for i in range(num_recvs): - final_i = done_evts[i].wait() - self.assertEqual(final_i, 0) + for evt in done_evts: + self.assertEqual(evt.wait(), 0) @skip_unless(zmq_supported) From c2b8e02927b6529d72c10dda06d2a19bea4e4268 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 26 Jan 2012 13:57:57 -0500 Subject: [PATCH 32/41] Avoid calling trampoline for zmq sockets. Instead leave a listener registered in the hub until the socket is closed. --- eventlet/green/zmq.py | 93 ++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index f1ebe95..f38ac5f 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -22,6 +22,7 @@ class _QueueLock(object): self._waiters = deque() self._count = 0 self._holder = None + self._hub = hubs.get_hub() def __nonzero__(self): return self._count @@ -37,7 +38,7 @@ class _QueueLock(object): if (self._waiters or self._count > 0) and self._holder is not current: # block until lock is free self._waiters.append(current) - hubs.get_hub().switch() + self._hub.switch() w = self._waiters.popleft() assert w is current, 'Waiting threads woken out of order' @@ -55,7 +56,7 @@ class _QueueLock(object): self._holder = None if self._waiters: # wake next - hubs.get_hub().schedule_call_global(0, self._waiters[0].switch) + self._hub.schedule_call_global(0, self._waiters[0].switch) class _SimpleEvent(object): """Represents a possibly blocked thread which may be blocked @@ -67,6 +68,7 @@ class _SimpleEvent(object): def __init__(self): self._blocked_thread = None self._wakeupper = None + self._hub = hubs.get_hub() def __nonzero__(self): return self._blocked_thread is not None @@ -95,7 +97,7 @@ class _SimpleEvent(object): blocked thread, then this call has no effect and returns False.""" if self._blocked_thread is not None and self._wakeupper is None: - self._wakeupper = hubs.get_hub().schedule_call_global(0, self._blocked_thread.switch) + self._wakeupper = self._hub.schedule_call_global(0, self._blocked_thread.switch) return True return False @@ -207,10 +209,10 @@ class Socket(_Socket): * recv_multipart """ + _event_listener = None def __init__(self, context, socket_type): - super(Socket, self).__init__(context, socket_type) + _Socket.__init__(self, context, socket_type) - self._in_trampoline = False self._send_event = _SimpleEvent() self._recv_event = _SimpleEvent() @@ -234,28 +236,25 @@ class Socket(_Socket): else: self.recv = self.recv_multipart = self._send_not_supported - def _trampoline(self, evt): - """Wait for events on the zmq socket. After this method - returns it is still possible that send and recv will return - EAGAIN. + def event(fd): + # Some events arrived at the zmq socket. This may mean + # there's a message that can be read or there's space for + # a message to be written. + self._send_event.wake() + self._recv_event.wake() - This supports being called by two separate greenthreads, a - sender and a receiver, but only the first caller will actually - call eventlet's trampoline method. The second thread will - still block. - """ + hub = hubs.get_hub() + self._event_listener = hub.add(hub.READ, self.getsockopt(FD), event) - if self._in_trampoline: - # Already a thread blocked in trampoline. - evt.block() - else: - try: - self._in_trampoline = True - with evt: - # Only trampoline on read events for zmq FDs, never write. - trampoline(self.getsockopt(FD), read=True) - finally: - self._in_trampoline = False + def close(self): + if self._event_listener is not None: + hubs.get_hub().remove(self._event_listener) + self._event_listener = None + # wake any blocked threads + self._send_event.wake() + self._recv_event.wake() + + _Socket.close(self) @_wraps(_Socket.send) def send(self, msg, flags=0, copy=True, track=False): @@ -273,7 +272,7 @@ class Socket(_Socket): return _Socket_send(self, msg, flags, copy, track) except ZMQError, e: if e.errno == EAGAIN: - self._trampoline(self._send_event) + self._send_event.block() else: raise @@ -293,7 +292,7 @@ class Socket(_Socket): return _Socket_recv(self, flags, copy, track) except ZMQError, e: if e.errno == EAGAIN: - self._trampoline(self._recv_event) + self._recv_event.block() else: raise @@ -306,10 +305,9 @@ class Socket(_Socket): # there is a greenthread blocked and waiting for events, # it will miss the edge-triggered read event, so wake it # up. - if self._send_evt and (result & POLLOUT): + if (result & POLLOUT): self._send_evt.wake() - - if self._recv_evt and (result & POLLIN): + if (result & POLLIN): self._recv_evt.wake() return result @@ -327,8 +325,11 @@ class Socket(_Socket): """ if flags & NOBLOCK: result = _Socket_send(self, msg, flags, copy, track) - if self._send_event or self._recv_event: - self.getsockopt(EVENTS) # triggers wakeups + # Instead of calling both wake methods, could call + # self.getsockopt(EVENTS) which would trigger wakeups if + # needed. + self._send_event.wake() + self._recv_event.wake() return result # TODO: pyzmq will copy the message buffer and create Message @@ -341,15 +342,14 @@ class Socket(_Socket): return _Socket_send(self, msg, flags, copy, track) except ZMQError, e: if e.errno == EAGAIN: - self._trampoline(self._send_event) + self._send_event.block() else: raise finally: # The call to send processes 0mq events and may # make the socket ready to recv. Wake the next # receiver. (Could check EVENTS for POLLIN here) - if self._recv_event: - self._recv_event.wake() + self._recv_event.wake() @_wraps(_Socket.send_multipart) @@ -374,27 +374,28 @@ class Socket(_Socket): """ if flags & NOBLOCK: msg = _Socket_recv(self, flags, copy, track) - if self._send_event or self._recv_event: - self.getsockopt(EVENTS) # triggers wakeups + # Instead of calling both wake methods, could call + # self.getsockopt(EVENTS) which would trigger wakeups if + # needed. + self._send_event.wake() + self._recv_event.wake() return msg flags |= NOBLOCK with self._recv_lock: while True: try: - try: - return _Socket_recv(self, flags, copy, track) - finally: - # The call to recv processes 0mq events and may - # make the socket ready to send. Wake the next - # receiver. (Could check EVENTS for POLLOUT here) - if self._send_event: - self._send_event.wake() + return _Socket_recv(self, flags, copy, track) except ZMQError, e: if e.errno == EAGAIN: - self._trampoline(self._recv_event) + self._recv_event.block() else: raise + finally: + # The call to recv processes 0mq events and may + # make the socket ready to send. Wake the next + # receiver. (Could check EVENTS for POLLOUT here) + self._send_event.wake() @_wraps(_Socket.recv_multipart) def _xsafe_recv_multipart(self, flags=0, copy=True, track=False): From 19ea50f69494cf6453ff2f8287a5bb11ed6053f3 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 26 Jan 2012 15:12:01 -0500 Subject: [PATCH 33/41] Rename zmq._SimpleEvent. Add close during recv test. Updating comments. --- eventlet/green/zmq.py | 77 ++++++++++++++++++------------------------- tests/zmq_test.py | 56 +++++++++++++------------------ 2 files changed, 54 insertions(+), 79 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index f38ac5f..42f0ced 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -58,12 +58,11 @@ class _QueueLock(object): # wake next self._hub.schedule_call_global(0, self._waiters[0].switch) -class _SimpleEvent(object): - """Represents a possibly blocked thread which may be blocked - inside this class' block method or inside a trampoline call. In - either case, the threads can be awoken by calling wake(). Wake() - can be called multiple times and all but the first call will have - no effect.""" +class _BlockedThread(object): + """Is either empty, or represents a single blocked thread that + blocked itself by calling the block() method. The thread can be + awoken by calling wake(). Wake() can be called multiple times and + all but the first call will have no effect.""" def __init__(self): self._blocked_thread = None @@ -74,22 +73,20 @@ class _SimpleEvent(object): return self._blocked_thread is not None def block(self): - with self: - hubs.get_hub().switch() - - def __enter__(self): if self._blocked_thread is not None: - raise Exception("Cannot block more than one thread on one SimpleEvent") + raise Exception("Cannot block more than one thread on one BlockedThread") self._blocked_thread = greenlet.getcurrent() - - def __exit__(self, type, value, traceback): - self._blocked_thread = None - # cleanup the wakeup task - if self._wakeupper is not None: - # Important to cancel the wakeup task so it doesn't - # spuriously wake this greenthread later on. - self._wakeupper.cancel() - self._wakeupper = None + + try: + self._hub.switch() + finally: + self._blocked_thread = None + # cleanup the wakeup task + if self._wakeupper is not None: + # Important to cancel the wakeup task so it doesn't + # spuriously wake this greenthread later on. + self._wakeupper.cancel() + self._wakeupper = None def wake(self): """Schedules the blocked thread to be awoken and return @@ -136,21 +133,11 @@ def _wraps(source_fn): # the 0mq context is associated with, which is the native thread the # greenthreads are running on, and the only operations that cause the # events to be read and processed are send(), recv() and -# getsockopt(EVENTS). This means that after doing any of these three -# operations, the ability of the socket to send or receive a message -# without blocking may have changed. If you're not careful, this can -# cause the hub to miss the read event for the socket. -# -# For example, suppose thread A calls trampoline and blocks because it -# called recv() when there was no waiting message. It should be -# notified when the state of the socket changes. However, while thread -# A is blocked, thread B calls send(), which internally causes the -# events to be processed, and the socket learns that it has a message -# waiting to be received. Unfortunately, because eventlet is currently -# running greenthread B, it isn't currently blocked in hub.wait() in -# poll or the equivalent. When hub.wait() is eventually called, the -# socket's event pipe will no longer be readable, so thread A will not -# be awoken, even though a message is waiting to be read! +# getsockopt(zmq.EVENTS). This means that after doing any of these +# three operations, the ability of the socket to send or receive a +# message without blocking may have changed, but after the events are +# read the FD is no longer readable so the hub may not signal our +# listener. # # If we understand that after calling send() a message might be ready # to be received and that after calling recv() a message might be able @@ -168,21 +155,16 @@ def _wraps(source_fn): # 2. Call getsockopt(zmq.EVENTS) and explicitly check if the other # thread should be woken up. This avoids spurious wake-ups but may # add overhead because getsockopt will cause all events to be -# processed, whereas send and recv can avoid processing +# processed, whereas send and recv throttle processing # events. Admittedly, all of the events will need to be processed # eventually, but it is likely faster to batch the processing. # -# Which approach is better? I have no idea. Right now the NOBLOCK -# paths in _xsafe_send and _xsafe_recv check getsockopt(zmq.EVENTS) -# and the other paths always wake the other blocked thread. It's done -# this way only because it was convenient to implement, not based on -# any benchmarks. +# Which approach is better? I have no idea. # # TODO: # - Ensure that recv* and send* methods raise error when called on a # closed socket. They should not block. -# - Return correct message tracker from send* methods -# - Make MessageTracker.wait zmq friendly +# - Support MessageTrackers and make MessageTracker.wait green # - What should happen to threads blocked on send/recv when socket is # closed? @@ -213,8 +195,8 @@ class Socket(_Socket): def __init__(self, context, socket_type): _Socket.__init__(self, context, socket_type) - self._send_event = _SimpleEvent() - self._recv_event = _SimpleEvent() + self._send_event = _BlockedThread() + self._recv_event = _BlockedThread() # customize send and recv methods based on socket type ops = self._eventlet_ops.get(socket_type) @@ -268,6 +250,11 @@ class Socket(_Socket): flags |= NOBLOCK while True: + # Avoid recreating a Message object each time Socket.send + # is called. TODO: Not sure if this is benefitial or not. + #if not copy and not track and type(msg) is str: + # msg = Message(msg) + try: return _Socket_send(self, msg, flags, copy, track) except ZMQError, e: diff --git a/tests/zmq_test.py b/tests/zmq_test.py index 67f9561..0c12d31 100644 --- a/tests/zmq_test.py +++ b/tests/zmq_test.py @@ -336,6 +336,26 @@ got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) final_i = done.wait() self.assertEqual(final_i, 0) + @skip_unless(zmq_supported) + def test_close_during_recv(self): + sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) + sleep() + done1 = event.Event() + done2 = event.Event() + + def rx(e): + self.assertRaisesErrno(zmq.ENOTSUP, receiver.recv) + e.send() + + spawn(rx, done1) + spawn(rx, done2) + + sleep() + receiver.close() + + done1.wait() + done2.wait() + class TestQueueLock(LimitedTestCase): @skip_unless(zmq_supported) def test_queue_lock_order(self): @@ -416,10 +436,10 @@ class TestQueueLock(LimitedTestCase): s.acquire() self.assertEquals(results, [1]) -class TestSimpleEvent(LimitedTestCase): +class TestBlockedThread(LimitedTestCase): @skip_unless(zmq_supported) def test_block(self): - e = zmq._SimpleEvent() + e = zmq._BlockedThread() done = event.Event() self.assertFalse(e) @@ -433,35 +453,3 @@ class TestSimpleEvent(LimitedTestCase): self.assertFalse(done.has_result()) e.wake() done.wait() - - @skip_unless(zmq_supported) - def test_enter_exit(self): - e = zmq._SimpleEvent() - done = event.Event() - self.assertFalse(e) - - def block(): - with e: - get_hub().switch() - done.send(1) - - gt = spawn(block) - sleep() - - self.assertFalse(done.has_result()) - get_hub().schedule_call_global(0, gt.switch) - done.wait() - - - @skip_unless(zmq_supported) - def test_error(self): - e1 = zmq._SimpleEvent() - with e1: - with self.assertRaises(Exception): - with e1: - pass - - e2 = zmq._SimpleEvent() - with e2: - with self.assertRaises(Exception): - e2.block() From bf88d5372e44e5b2cdb4b8ff0c359e38bc500cfa Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 26 Jan 2012 15:18:19 -0500 Subject: [PATCH 34/41] test closing zmq x-sockets --- eventlet/green/zmq.py | 4 ---- tests/zmq_test.py | 9 +++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 42f0ced..5687f2e 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -162,11 +162,7 @@ def _wraps(source_fn): # Which approach is better? I have no idea. # # TODO: -# - Ensure that recv* and send* methods raise error when called on a -# closed socket. They should not block. # - Support MessageTrackers and make MessageTracker.wait green -# - What should happen to threads blocked on send/recv when socket is -# closed? _Socket = __zmq__.Socket _Socket_recv = _Socket.recv diff --git a/tests/zmq_test.py b/tests/zmq_test.py index 0c12d31..df9c4ab 100644 --- a/tests/zmq_test.py +++ b/tests/zmq_test.py @@ -75,6 +75,15 @@ got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) self.assertRaisesErrno(zmq.ENOTSUP, rep.recv) self.assertRaisesErrno(zmq.ENOTSUP, req.send, 'test') + @skip_unless(zmq_supported) + def test_close_xsocket_raises_enotsup(self): + req, rep, port = self.create_bound_pair(zmq.XREQ, zmq.XREP) + + rep.close() + req.close() + self.assertRaisesErrno(zmq.ENOTSUP, rep.recv) + self.assertRaisesErrno(zmq.ENOTSUP, req.send, 'test') + @skip_unless(zmq_supported) def test_send_1k_req_rep(self): req, rep, port = self.create_bound_pair(zmq.REQ, zmq.REP) From 4b749a2c13449c5ed772d13b6867071cd910f2d5 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 26 Jan 2012 15:31:20 -0500 Subject: [PATCH 35/41] wrap all zmq send and recv methods to use correct docs and names --- eventlet/green/zmq.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 5687f2e..4eff5c2 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -143,12 +143,12 @@ def _wraps(source_fn): # to be received and that after calling recv() a message might be able # to be sent, what should we do next? There are two approaches: # -# 1. Always wake the other thread if there is one waiting. This -# wakeup may be spurious because the socket might not actually be -# ready for a send() or recv(). However, if a thread is in a -# tight-loop successfully calling send() or recv() then the wakeups -# are naturally batched and there's very little cost added to each -# send/recv call. +# 1. Always wake the other thread if there is one waiting. This +# wakeup may be spurious because the socket might not actually be +# ready for a send() or recv(). However, if a thread is in a +# tight-loop successfully calling send() or recv() then the wakeups +# are naturally batched and there's very little cost added to each +# send/recv call. # # or # @@ -224,6 +224,7 @@ class Socket(_Socket): hub = hubs.get_hub() self._event_listener = hub.add(hub.READ, self.getsockopt(FD), event) + @_wraps(_Socket.close) def close(self): if self._event_listener is not None: hubs.get_hub().remove(self._event_listener) @@ -294,9 +295,11 @@ class Socket(_Socket): self._recv_evt.wake() return result + @_wraps(_Socket.send) def _send_not_supported(self, msg, flags, copy, track): raise ZMQError(ENOTSUP) + @_wraps(_Socket.recv) def _recv_not_supported(self, flags, copy, track): raise ZMQError(ENOTSUP) From 1f21af4843dd09402f9ed0053e1136d2b94ffb43 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 26 Jan 2012 15:37:02 -0500 Subject: [PATCH 36/41] fix zmq test that sometimes called recv on closed skt --- tests/zmq_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/zmq_test.py b/tests/zmq_test.py index df9c4ab..b1a40cc 100644 --- a/tests/zmq_test.py +++ b/tests/zmq_test.py @@ -96,14 +96,13 @@ got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) while req.recv() != 'done': tx_i += 1 req.send(str(tx_i)) + done.send(0) def rx(): while True: rx_i = rep.recv() if rx_i == "1000": rep.send('done') - sleep() - done.send(0) break rep.send('i') spawn(tx) From 692cc5dcef1e90091a1f4f102a25b0024acd825d Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 26 Jan 2012 16:58:38 -0500 Subject: [PATCH 37/41] ensure sockets are collected and closed in zmq test --- tests/zmq_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/zmq_test.py b/tests/zmq_test.py index b1a40cc..107a339 100644 --- a/tests/zmq_test.py +++ b/tests/zmq_test.py @@ -19,7 +19,9 @@ def zmq_supported(_): class TestUpstreamDownStream(LimitedTestCase): - sockets = [] + def setUp(self): + super(TestUpstreamDownStream, self).setUp() + self.sockets = [] def tearDown(self): self.clear_up_sockets() @@ -32,12 +34,14 @@ class TestUpstreamDownStream(LimitedTestCase): port = s1.bind_to_random_port(interface) s2 = context.socket(type2) s2.connect('%s:%s' % (interface, port)) - self.sockets = [s1, s2] + self.sockets.append(s1) + self.sockets.append(s2) return s1, s2, port def clear_up_sockets(self): for sock in self.sockets: sock.close() + self.sockets = None def assertRaisesErrno(self, errno, func, *args): try: From 8b570da4b7cd13bb8cb574c5a7ad7b97f5f82434 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 26 Jan 2012 17:24:13 -0500 Subject: [PATCH 38/41] mistakenly changed version_info. change it back. --- eventlet/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eventlet/__init__.py b/eventlet/__init__.py index 9d92c75..2e47faf 100644 --- a/eventlet/__init__.py +++ b/eventlet/__init__.py @@ -1,4 +1,4 @@ -version_info = (0, 9, 16, 1) +version_info = (0, 9, 17, "dev") __version__ = ".".join(map(str, version_info)) try: From 36273a3483babe21e01b737b124552350fbaf3cc Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 26 Jan 2012 17:32:12 -0500 Subject: [PATCH 39/41] remove unused import --- eventlet/green/zmq.py | 1 - 1 file changed, 1 deletion(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 4eff5c2..4db42f0 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -2,7 +2,6 @@ """ __zmq__ = __import__('zmq') from eventlet import hubs -from eventlet.hubs import trampoline from eventlet.patcher import slurp_properties from eventlet.support import greenlets as greenlet From c3953d4b602d303c6150667c64ad5044ba7fb164 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Thu, 2 Feb 2012 19:18:30 -0500 Subject: [PATCH 40/41] Simplify zmq. Remove premature optimizations. Add "_eventlet" to all new attributes. --- eventlet/green/zmq.py | 161 ++++++++---------------------------------- 1 file changed, 30 insertions(+), 131 deletions(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 4db42f0..58f6b53 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -185,99 +185,33 @@ class Socket(_Socket): * send_multipart * recv_multipart """ - - _event_listener = None def __init__(self, context, socket_type): - _Socket.__init__(self, context, socket_type) + super(Socket, self).__init__(context, socket_type) - self._send_event = _BlockedThread() - self._recv_event = _BlockedThread() - - # customize send and recv methods based on socket type - ops = self._eventlet_ops.get(socket_type) - if ops: - self._send_lock = None - self._recv_lock = None - send, msend, recv, mrecv = ops - if send: - self._send_lock = _QueueLock() - self.send = MethodType(send, self, Socket) - self.send_multipart = MethodType(msend, self, Socket) - else: - self.send = self.send_multipart = self._send_not_supported - - if recv: - self._recv_lock = _QueueLock() - self.recv = MethodType(recv, self, Socket) - self.recv_multipart = MethodType(mrecv, self, Socket) - else: - self.recv = self.recv_multipart = self._send_not_supported + self._eventlet_send_event = _BlockedThread() + self._eventlet_recv_event = _BlockedThread() + self._eventlet_send_lock = _QueueLock() + self._eventlet_recv_lock = _QueueLock() def event(fd): # Some events arrived at the zmq socket. This may mean # there's a message that can be read or there's space for # a message to be written. - self._send_event.wake() - self._recv_event.wake() + self._eventlet_send_event.wake() + self._eventlet_recv_event.wake() hub = hubs.get_hub() - self._event_listener = hub.add(hub.READ, self.getsockopt(FD), event) + self._eventlet_listener = hub.add(hub.READ, self.getsockopt(FD), event) @_wraps(_Socket.close) def close(self): - if self._event_listener is not None: - hubs.get_hub().remove(self._event_listener) - self._event_listener = None - # wake any blocked threads - self._send_event.wake() - self._recv_event.wake() - _Socket.close(self) - - @_wraps(_Socket.send) - def send(self, msg, flags=0, copy=True, track=False): - """Send method used by REP and REQ sockets. The lock-step - send->recv->send->recv restriction of these sockets makes this - implementation simple. - """ - if flags & NOBLOCK: - return _Socket_send(self, msg, flags, copy, track) - - flags |= NOBLOCK - - while True: - # Avoid recreating a Message object each time Socket.send - # is called. TODO: Not sure if this is benefitial or not. - #if not copy and not track and type(msg) is str: - # msg = Message(msg) - - try: - return _Socket_send(self, msg, flags, copy, track) - except ZMQError, e: - if e.errno == EAGAIN: - self._send_event.block() - else: - raise - - @_wraps(_Socket.recv) - def recv(self, flags=0, copy=True, track=False): - """Recv method used by REP and REQ sockets. The lock-step - send->recv->send->recv restriction of these sockets makes this - implementation simple. - """ - if flags & NOBLOCK: - return _Socket_recv(self, flags, copy, track) - - flags |= NOBLOCK - - while True: - try: - return _Socket_recv(self, flags, copy, track) - except ZMQError, e: - if e.errno == EAGAIN: - self._recv_event.block() - else: - raise + if self._eventlet_listener is not None: + hubs.get_hub().remove(self._eventlet_listener) + self._eventlet_listener = None + # wake any blocked threads + self._eventlet_send_event.wake() + self._eventlet_recv_event.wake() @_wraps(_Socket.getsockopt) def getsockopt(self, option): @@ -295,15 +229,7 @@ class Socket(_Socket): return result @_wraps(_Socket.send) - def _send_not_supported(self, msg, flags, copy, track): - raise ZMQError(ENOTSUP) - - @_wraps(_Socket.recv) - def _recv_not_supported(self, flags, copy, track): - raise ZMQError(ENOTSUP) - - @_wraps(_Socket.send) - def _xsafe_send(self, msg, flags=0, copy=True, track=False): + def send(self, msg, flags=0, copy=True, track=False): """A send method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. @@ -313,32 +239,32 @@ class Socket(_Socket): # Instead of calling both wake methods, could call # self.getsockopt(EVENTS) which would trigger wakeups if # needed. - self._send_event.wake() - self._recv_event.wake() + self._eventlet_send_event.wake() + self._eventlet_recv_event.wake() return result # TODO: pyzmq will copy the message buffer and create Message # objects under some circumstances. We could do that work here # once to avoid doing it every time the send is retried. flags |= NOBLOCK - with self._send_lock: + with self._eventlet_send_lock: while True: try: return _Socket_send(self, msg, flags, copy, track) except ZMQError, e: if e.errno == EAGAIN: - self._send_event.block() + self._eventlet_send_event.block() else: raise finally: # The call to send processes 0mq events and may # make the socket ready to recv. Wake the next # receiver. (Could check EVENTS for POLLIN here) - self._recv_event.wake() + self._eventlet_recv_event.wake() @_wraps(_Socket.send_multipart) - def _xsafe_send_multipart(self, msg_parts, flags=0, copy=True, track=False): + def send_multipart(self, msg_parts, flags=0, copy=True, track=False): """A send_multipart method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. @@ -348,11 +274,11 @@ class Socket(_Socket): # acquire lock here so the subsequent calls to send for the # message parts after the first don't block - with self._send_lock: + with self._eventlet_send_lock: return _Socket_send_multipart(self, msg_parts, flags, copy, track) @_wraps(_Socket.recv) - def _xsafe_recv(self, flags=0, copy=True, track=False): + def recv(self, flags=0, copy=True, track=False): """A recv method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. @@ -362,28 +288,28 @@ class Socket(_Socket): # Instead of calling both wake methods, could call # self.getsockopt(EVENTS) which would trigger wakeups if # needed. - self._send_event.wake() - self._recv_event.wake() + self._eventlet_send_event.wake() + self._eventlet_recv_event.wake() return msg flags |= NOBLOCK - with self._recv_lock: + with self._eventlet_recv_lock: while True: try: return _Socket_recv(self, flags, copy, track) except ZMQError, e: if e.errno == EAGAIN: - self._recv_event.block() + self._eventlet_recv_event.block() else: raise finally: # The call to recv processes 0mq events and may # make the socket ready to send. Wake the next # receiver. (Could check EVENTS for POLLOUT here) - self._send_event.wake() + self._eventlet_send_event.wake() @_wraps(_Socket.recv_multipart) - def _xsafe_recv_multipart(self, flags=0, copy=True, track=False): + def recv_multipart(self, flags=0, copy=True, track=False): """A recv_multipart method that's safe to use when multiple greenthreads are calling send, send_multipart, recv and recv_multipart on the same socket. @@ -393,32 +319,5 @@ class Socket(_Socket): # acquire lock here so the subsequent calls to recv for the # message parts after the first don't block - with self._recv_lock: + with self._eventlet_recv_lock: return _Socket_recv_multipart(self, flags, copy, track) - - # The behavior of the send and recv methods depends on the socket - # type. See http://api.zeromq.org/2-1:zmq-socket for explanation - # of socket types. For the green Socket, our main concern is - # supporting calling send or recv from multiple greenthreads when - # it makes sense for the socket type. - _send_only_ops = (_xsafe_send, _xsafe_send_multipart, None, None) - _recv_only_ops = (None, None, _xsafe_recv, _xsafe_recv_multipart) - _full_ops = (_xsafe_send, _xsafe_send_multipart, _xsafe_recv, _xsafe_recv_multipart) - - _eventlet_ops = { - PUB: _send_only_ops, - SUB: _recv_only_ops, - - PUSH: _send_only_ops, - PULL: _recv_only_ops, - - PAIR: _full_ops - } - - try: - _eventlet_ops[XREP] = _full_ops - _eventlet_ops[XREQ] = _full_ops - except AttributeError: - # XREP and XREQ are being renamed ROUTER and DEALER - _eventlet_ops[ROUTER] = _full_ops - _eventlet_ops[DEALER] = _full_ops From de41585d2b7858b2d36882baaec9101cdea141f9 Mon Sep 17 00:00:00 2001 From: Geoff Salmon Date: Mon, 6 Feb 2012 08:18:09 -0500 Subject: [PATCH 41/41] unused import --- eventlet/green/zmq.py | 1 - 1 file changed, 1 deletion(-) diff --git a/eventlet/green/zmq.py b/eventlet/green/zmq.py index 58f6b53..5f379e7 100644 --- a/eventlet/green/zmq.py +++ b/eventlet/green/zmq.py @@ -9,7 +9,6 @@ __patched__ = ['Context', 'Socket'] slurp_properties(__zmq__, globals(), ignore=__patched__) from collections import deque -from types import MethodType class _QueueLock(object): """A Lock that can be acquired by at most one thread. Any other