openstack/deb-python-eventlet
Code Issues Proposed changes

Added eventlet.green.OpenSSL to further rationalize our SSL support, carved off ssl tests into ssl_test.py, and added some docs on coverage.

Browse Source
This commit is contained in:
Ryan Williams
2009-11-27 23:56:30 -05:00
parent 986bed61d2
commit a25b889a52
17 changed files with 342 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.hgignore
View File

@@ -9,4 +9,5 @@ htmlreports
*.esproj
.DS_Store
results.*.db
doc/_build
doc/_build
annotated

18
doc/testing.rst
View File

@@ -79,4 +79,22 @@ If you are writing a test that involves a client connecting to a spawned server,
server_sock = api.tcp_listener(('127.0.0.1', 0))
client_sock = api.connect_tcp(('localhost', server_sock.getsockname()[1]))
Coverage
--------
Coverage.py is an awesome tool for evaluating how much code was exercised by unit tests. Nose supports it if both are installed, so it's easy to generate coverage reports for eventlet. Here's how:
.. code-block:: sh
nosetests --with-coverage
After running the tests to completion, this will emit a huge wodge of module names and line numbers. For some reason, the ``--cover-inclusive`` option breaks everything rather than serving its purpose of limiting the coverage to the local files, so don't use that.
The annotate option is quite useful because it generates annotated source files that are much easier to read than line-number soup. Here's a command that runs the annotation, dumping the annotated files into a directory called "annotated":
.. code-block:: sh
coverage annotate -d annotated --omit tempmod
(``tempmod`` is omitted because it gets thrown away at the completion of its unit test and coverage.py isn't smart enough to detect this)

186
eventlet/green/OpenSSL/SSL.py Normal file
View File

@@ -0,0 +1,186 @@
from OpenSSL import SSL as orig_SSL
from OpenSSL.SSL import *
from eventlet import greenio
from eventlet.api import trampoline
import socket
class GreenConnection(greenio.GreenSocket):
""" Nonblocking wrapper for SSL.Connection objects.
"""
def __init__(self, ctx, sock=None):
if sock is not None:
fd = orig_SSL.Connection(ctx, sock)
else:
# if we're given a Connection object directly, use it;
# this is used in the inherited accept() method
fd = ctx
super(ConnectionType, self).__init__(fd)
self.sock = self
def close(self):
super(GreenConnection, self).close()
def do_handshake(self):
""" Perform an SSL handshake (usually called after renegotiate or one of
set_accept_state or set_accept_state). This can raise the same exceptions as
send and recv. """
if self.act_non_blocking:
return self.fd.do_handshake()
while True:
try:
return self.fd.do_handshake()
except WantReadError:
trampoline(self.fd.fileno(),
read=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
except WantWriteError:
trampoline(self.fd.fileno(),
write=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
def dup(self):
raise NotImplementedError("Dup not supported on SSL sockets")
def get_app_data(self, *args, **kw):
fn = self.get_app_data = self.fd.get_app_data
return fn(*args, **kw)
def set_app_data(self, *args, **kw):
fn = self.set_app_data = self.fd.set_app_data
return fn(*args, **kw)
def get_cipher_list(self, *args, **kw):
fn = self.get_cipher_list = self.fd.get_cipher_list
return fn(*args, **kw)
def get_context(self, *args, **kw):
fn = self.get_context = self.fd.get_context
return fn(*args, **kw)
def get_peer_certificate(self, *args, **kw):
fn = self.get_peer_certificate = self.fd.get_peer_certificate
return fn(*args, **kw)
def makefile(self, mode='r', bufsize=-1):
raise NotImplementedError("Makefile not supported on SSL sockets")
def pending(self, *args, **kw):
fn = self.pending = self.fd.pending
return fn(*args, **kw)
def read(self, size):
"""Works like a blocking call to SSL_read(), whose behavior is
described here: http://www.openssl.org/docs/ssl/SSL_read.html"""
if self.act_non_blocking:
return self.fd.read(size)
while True:
try:
return self.fd.read(size)
except WantReadError:
trampoline(self.fd.fileno(),
read=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
except WantWriteError:
trampoline(self.fd.fileno(),
write=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
except SysCallError, e:
if e[0] == -1 or e[0] > 0:
return ''
recv = read
def renegotiate(self, *args, **kw):
fn = self.renegotiate = self.fd.renegotiate
return fn(*args, **kw)
def write(self, data):
"""Works like a blocking call to SSL_write(), whose behavior is
described here: http://www.openssl.org/docs/ssl/SSL_write.html"""
if not data:
return 0 # calling SSL_write() with 0 bytes to be sent is undefined
if self.act_non_blocking:
return self.fd.write(data)
while True:
try:
return self.fd.write(data)
except WantReadError:
trampoline(self.fd.fileno(),
read=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
except WantWriteError:
trampoline(self.fd.fileno(),
write=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
send = write
def sendall(self, data):
"""Send "all" data on the connection. This calls send() repeatedly until
all data is sent. If an error occurs, it's impossible to tell how much data
has been sent.
No return value."""
tail = self.send(data)
while tail < len(data):
tail += self.send(data[tail:])
def set_accept_state(self, *args, **kw):
fn = self.set_accept_state = self.fd.set_accept_state
return fn(*args, **kw)
def set_connect_state(self, *args, **kw):
fn = self.set_connect_state = self.fd.set_connect_state
return fn(*args, **kw)
def shutdown(self):
if self.act_non_blocking:
return self.fd.shutdown()
while True:
try:
return self.fd.shutdown()
except WantReadError:
trampoline(self.fd.fileno(),
read=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
except WantWriteError:
trampoline(self.fd.fileno(),
write=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
def get_shutdown(self, *args, **kw):
fn = self.get_shutdown = self.fd.get_shutdown
return fn(*args, **kw)
def set_shutdown(self, *args, **kw):
fn = self.set_shutdown = self.fd.set_shutdown
return fn(*args, **kw)
def sock_shutdown(self, *args, **kw):
fn = self.sock_shutdown = self.fd.sock_shutdown
return fn(*args, **kw)
def state_string(self, *args, **kw):
fn = self.state_string = self.fd.state_string
return fn(*args, **kw)
def want_read(self, *args, **kw):
fn = self.want_read = self.fd.want_read
return fn(*args, **kw)
def want_write(self, *args, **kw):
fn = self.want_write = self.fd.want_write
return fn(*args, **kw)
Connection = ConnectionType = GreenConnection
del greenio

2
eventlet/green/OpenSSL/__init__.py Normal file
View File

@@ -0,0 +1,2 @@
import rand, crypto, SSL, tsafe
from version import __version__

1
eventlet/green/OpenSSL/crypto.py Normal file
View File

@@ -0,0 +1 @@
from OpenSSL.crypto import *

1
eventlet/green/OpenSSL/rand.py Normal file
View File

@@ -0,0 +1 @@
from OpenSSL.rand import *

1
eventlet/green/OpenSSL/tsafe.py Normal file
View File

@@ -0,0 +1 @@
from OpenSSL.tsafe import *

1
eventlet/green/OpenSSL/version.py Normal file
View File

@@ -0,0 +1 @@
from OpenSSL.version import __version__, __doc__

9
eventlet/green/socket.py
View File

@@ -5,7 +5,8 @@ _fileobject = __socket._fileobject
from eventlet.api import get_hub
from eventlet.greenio import GreenSocket as socket
from eventlet.greenio import SSL as _SSL
from eventlet.greenio import SSL as _SSL # for exceptions
import warnings
def fromfd(*args):
return socket(__socket.fromfd(*args))
@@ -127,13 +128,13 @@ class GreenSSLObject(object):
try:
try:
# >= Python 2.6
from eventlet.green import ssl
from eventlet.green import ssl as ssl_module
sslerror = __socket.sslerror
__socket.ssl
def ssl(sock, certificate=None, private_key=None):
warnings.warn("socket.ssl() is deprecated. Use ssl.wrap_socket() instead.",
DeprecationWarning, stacklevel=2)
return ssl.sslwrap_simple(sock, keyfile, certfile)
return ssl_module.sslwrap_simple(sock, private_key, certificate)
except ImportError:
# <= Python 2.5 compatibility
sslerror = __socket.sslerror
@@ -144,5 +145,5 @@ try:
return GreenSSLObject(wrapped)
except AttributeError:
# if the real socket module doesn't have the ssl method or sslerror
# exception, it hardly seems useful to emulate them
# exception, we can't emulate them
pass

18
eventlet/green/ssl.py
View File

@@ -286,10 +286,14 @@ def wrap_socket(sock, keyfile=None, certfile=None,
suppress_ragged_eofs=suppress_ragged_eofs)
def sslwrap_simple(sock, keyfile=None, certfile=None):
"""A replacement for the old socket.ssl function. Designed
for compability with Python 2.5 and earlier. Will disappear in
Python 3.0."""
ssl_sock = GreenSSLSocket(sock, 0, keyfile, certfile, CERT_NONE,
PROTOCOL_SSLv23, None)
return ssl_sock
if hasattr(__ssl, 'sslwrap_simple'):
def sslwrap_simple(sock, keyfile=None, certfile=None):
"""A replacement for the old socket.ssl function. Designed
for compability with Python 2.5 and earlier. Will disappear in
Python 3.0."""
ssl_sock = GreenSSLSocket(sock, keyfile=keyfile, certfile=certfile,
server_side=False,
cert_reqs=CERT_NONE,
ssl_version=PROTOCOL_SSLv23,
ca_certs=None)
return ssl_sock

203
eventlet/greenio.py
View File

@@ -8,6 +8,7 @@ import socket
from socket import socket as _original_socket
import sys
import time
import warnings
from errno import EWOULDBLOCK, EAGAIN
@@ -497,204 +498,36 @@ class GreenPipe(GreenFile):
self.fd.fd.flush()
# backwards compatibility with old GreenSSL stuff
try:
from OpenSSL import SSL
def GreenSSL(fd):
assert isinstance(fd, (SSL.ConnectionType)), \
"GreenSSL must be constructed with an "\
"OpenSSL Connection object"
warnings.warn("GreenSSL is deprecated, please use "\
"eventlet.green.OpenSSL.Connection instead (if on "\
"Python 2.5) or eventlet.green.ssl.wrap_socket() "\
"(if on Python 2.6 or later)",
DeprecationWarning, stacklevel=2)
import eventlet.green.OpenSSL.SSL
return eventlet.green.OpenSSL.SSL.Connection(None, fd)
except ImportError:
# pyOpenSSL not installed, define exceptions anyway for convenience
class SSL(object):
class WantWriteError(object):
pass
class WantReadError(object):
pass
class ZeroReturnError(object):
pass
class SysCallError(object):
pass
class GreenSSL(GreenSocket):
""" Nonblocking wrapper for SSL.Connection objects.
Note: not compatible with SSLObject
(http://www.python.org/doc/2.5.2/lib/ssl-objects.html) because it does not
implement server() or issuer(), and the read() method has a mandatory size.
"""
def __init__(self, fd):
super(GreenSSL, self).__init__(fd)
assert isinstance(fd, (SSL.ConnectionType)), \
"GreenSSL can only be constructed with an "\
"OpenSSL Connection object"
self.sock = self
def close(self):
# *NOTE: in older versions of eventlet, we called shutdown() on SSL sockets
# before closing them. That wasn't right because correctly-written clients
# would have already called shutdown, and calling shutdown a second time
# triggers unwanted bidirectional communication.
super(GreenSSL, self).close()
def do_handshake(self):
""" Perform an SSL handshake (usually called after renegotiate or one of
set_accept_state or set_accept_state). This can raise the same exceptions as
send and recv. """
if self.act_non_blocking:
return self.fd.do_handshake()
while True:
try:
return self.fd.do_handshake()
except SSL.WantReadError:
trampoline(self.fd.fileno(),
read=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
except SSL.WantWriteError:
trampoline(self.fd.fileno(),
write=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
def dup(self):
raise NotImplementedError("Dup not supported on SSL sockets")
def get_app_data(self, *args, **kw):
fn = self.get_app_data = self.fd.get_app_data
return fn(*args, **kw)
def set_app_data(self, *args, **kw):
fn = self.set_app_data = self.fd.set_app_data
return fn(*args, **kw)
def get_cipher_list(self, *args, **kw):
fn = self.get_cipher_list = self.fd.get_cipher_list
return fn(*args, **kw)
def get_context(self, *args, **kw):
fn = self.get_context = self.fd.get_context
return fn(*args, **kw)
def get_peer_certificate(self, *args, **kw):
fn = self.get_peer_certificate = self.fd.get_peer_certificate
return fn(*args, **kw)
def makefile(self, mode='r', bufsize=-1):
raise NotImplementedError("Makefile not supported on SSL sockets")
def pending(self, *args, **kw):
fn = self.pending = self.fd.pending
return fn(*args, **kw)
def read(self, size):
"""Works like a blocking call to SSL_read(), whose behavior is
described here: http://www.openssl.org/docs/ssl/SSL_read.html"""
if self.act_non_blocking:
return self.fd.read(size)
while True:
try:
return self.fd.read(size)
except SSL.WantReadError:
trampoline(self.fd.fileno(),
read=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
except SSL.WantWriteError:
trampoline(self.fd.fileno(),
write=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
except SSL.SysCallError, e:
if e[0] == -1 or e[0] > 0:
return ''
recv = read
def renegotiate(self, *args, **kw):
fn = self.renegotiate = self.fd.renegotiate
return fn(*args, **kw)
def write(self, data):
"""Works like a blocking call to SSL_write(), whose behavior is
described here: http://www.openssl.org/docs/ssl/SSL_write.html"""
if not data:
return 0 # calling SSL_write() with 0 bytes to be sent is undefined
if self.act_non_blocking:
return self.fd.write(data)
while True:
try:
return self.fd.write(data)
except SSL.WantReadError:
trampoline(self.fd.fileno(),
read=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
except SSL.WantWriteError:
trampoline(self.fd.fileno(),
write=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
send = write
def sendall(self, data):
"""Send "all" data on the connection. This calls send() repeatedly until
all data is sent. If an error occurs, it's impossible to tell how much data
has been sent.
No return value."""
tail = self.send(data)
while tail < len(data):
tail += self.send(data[tail:])
def set_accept_state(self, *args, **kw):
fn = self.set_accept_state = self.fd.set_accept_state
return fn(*args, **kw)
def set_connect_state(self, *args, **kw):
fn = self.set_connect_state = self.fd.set_connect_state
return fn(*args, **kw)
def shutdown(self):
if self.act_non_blocking:
return self.fd.shutdown()
while True:
try:
return self.fd.shutdown()
except SSL.WantReadError:
trampoline(self.fd.fileno(),
read=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
except SSL.WantWriteError:
trampoline(self.fd.fileno(),
write=True,
timeout=self.timeout,
timeout_exc=socket.timeout)
def get_shutdown(self, *args, **kw):
fn = self.get_shutdown = self.fd.get_shutdown
return fn(*args, **kw)
def set_shutdown(self, *args, **kw):
fn = self.set_shutdown = self.fd.set_shutdown
return fn(*args, **kw)
def sock_shutdown(self, *args, **kw):
fn = self.sock_shutdown = self.fd.sock_shutdown
return fn(*args, **kw)
def state_string(self, *args, **kw):
fn = self.state_string = self.fd.state_string
return fn(*args, **kw)
def want_read(self, *args, **kw):
fn = self.want_read = self.fd.want_read
return fn(*args, **kw)
def want_write(self, *args, **kw):
fn = self.want_write = self.fd.want_write
return fn(*args, **kw)
def shutdown_safe(sock):
""" Shuts down the socket. This is a convenience method for

4
eventlet/util.py
View File

@@ -51,7 +51,7 @@ except ImportError:
# if ssl is not available, use PyOpenSSL
def wrap_ssl(sock, certificate=None, private_key=None, server_side=False):
try:
from OpenSSL import SSL
from eventlet.green.OpenSSL import SSL
except ImportError:
raise ImportError("To use SSL with Eventlet, you must install PyOpenSSL or use Python 2.6 or later.")
context = SSL.Context(SSL.SSLv23_METHOD)
@@ -66,7 +66,7 @@ except ImportError:
connection.set_accept_state()
else:
connection.set_connect_state()
return greenio.GreenSSL(connection)
return connection
socket_already_wrapped = False
def wrap_socket_with_coroutine_socket(use_thread_pool=True):

28
tests/__init__.py
View File

@@ -21,16 +21,24 @@ def skipped(func):
return skipme
def skip_unless_requirement(requirement):
def skip_unless(requirement):
""" Decorator that skips a test if the *requirement* does not return True.
*requirement* is a callable that accepts one argument, the function to be decorated,
and returns True if the requirement is satisfied.
*requirement* can be a boolean or a callable that accepts one argument.
The callable will be called with the function to be decorated, and
should return True if the requirement is satisfied.
"""
def skipped_wrapper(func):
if not requirement(func):
return skipped(func)
else:
return func
if isinstance(requirement, bool):
def skipped_wrapper(func):
if not requirement:
return skipped(func)
else:
return func
else:
def skipped_wrapper(func):
if not requirement(func):
return skipped(func)
else:
return func
return skipped_wrapper
@@ -42,7 +50,7 @@ def requires_twisted(func):
return 'Twisted' in type(get_hub()).__name__
except Exception:
return False
return skip_unless_requirement(requirement)(func)
return skip_unless(requirement)(func)
def skip_with_libevent(func):
@@ -50,7 +58,7 @@ def skip_with_libevent(func):
def requirement(_f):
from eventlet.api import get_hub
return not('libevent' in type(get_hub()).__module__)
return skip_unless_requirement(requirement)(func)
return skip_unless(requirement)(func)
class TestIsTakingTooLong(Exception):

4
tests/db_pool_test.py
View File

@@ -1,6 +1,6 @@
"Test cases for db_pool"
from tests import skipped, skip_unless_requirement
from tests import skipped, skip_unless
from unittest import TestCase, main
from eventlet import api, coros
from eventlet import db_pool
@@ -506,7 +506,7 @@ def mysql_requirement(_f):
class TestMysqlConnectionPool(object):
__test__ = True
@skip_unless_requirement(mysql_requirement)
@skip_unless(mysql_requirement)
def setUp(self):
import MySQLdb
self._dbmodule = MySQLdb

55
tests/greenio_test.py
View File

@@ -250,61 +250,6 @@ class TestGreenIo(LimitedTestCase):
finally:
sys.stderr = orig
self.assert_('Traceback' in fake.getvalue())
class SSLTest(LimitedTestCase):
def setUp(self):
super(SSLTest, self).setUp()
self.certificate_file = os.path.join(os.path.dirname(__file__), 'test_server.crt')
self.private_key_file = os.path.join(os.path.dirname(__file__), 'test_server.key')
def test_duplex_response(self):
def serve(listener):
sock, addr = listener.accept()
stuff = sock.read(8192)
sock.write('response')
sock = api.ssl_listener(('127.0.0.1', 0), self.certificate_file, self.private_key_file)
server_coro = coros.execute(serve, sock)
client = util.wrap_ssl(api.connect_tcp(('127.0.0.1', sock.getsockname()[1])))
client.write('line 1\r\nline 2\r\n\r\n')
self.assertEquals(client.read(8192), 'response')
server_coro.wait()
def test_greensslobject(self):
def serve(listener):
sock, addr = listener.accept()
sock.write('content')
greenio.shutdown_safe(sock)
sock.close()
listener = api.ssl_listener(('', 0),
self.certificate_file,
self.private_key_file)
killer = api.spawn(serve, listener)
client = util.wrap_ssl(api.connect_tcp(('localhost', listener.getsockname()[1])))
client = GreenSSLObject(client)
self.assertEquals(client.read(1024), 'content')
self.assertEquals(client.read(1024), '')
def test_ssl_close(self):
def serve(listener):
sock, addr = listener.accept()
stuff = sock.read(8192)
try:
self.assertEquals("", sock.read(8192))
except greenio.SSL.ZeroReturnError:
pass
sock = api.ssl_listener(('127.0.0.1', 0), self.certificate_file, self.private_key_file)
server_coro = coros.execute(serve, sock)
raw_client = api.connect_tcp(('127.0.0.1', sock.getsockname()[1]))
client = util.wrap_ssl(raw_client)
client.write('X')
greenio.shutdown_safe(client)
client.close()
server_coro.wait()
if __name__ == '__main__':
main()

8
tests/saranwrap_test.py
View File

@@ -227,14 +227,14 @@ class TestSaranwrap(unittest.TestCase):
def test_not_inheriting_pythonpath(self):
# construct a fake module in the temp directory
temp_dir = tempfile.mkdtemp("saranwrap_test")
fp = open(os.path.join(temp_dir, "jitar_hero.py"), "w")
fp = open(os.path.join(temp_dir, "tempmod.py"), "w")
fp.write("""import os, sys
pypath = os.environ['PYTHONPATH']
sys_path = sys.path""")
fp.close()
# this should fail because we haven't stuck the temp_dir in our path yet
prox = saranwrap.wrap_module('jitar_hero')
prox = saranwrap.wrap_module('tempmod')
try:
prox.pypath
self.fail()
@@ -244,8 +244,8 @@ sys_path = sys.path""")
# now try to saranwrap it
sys.path.append(temp_dir)
try:
import jitar_hero
prox = saranwrap.wrap(jitar_hero)
import tempmod
prox = saranwrap.wrap(tempmod)
self.assert_(prox.pypath.count(temp_dir))
self.assert_(prox.sys_path.count(temp_dir))
finally:

70
tests/ssl_test.py Normal file
View File

@@ -0,0 +1,70 @@
from tests import skipped, LimitedTestCase, skip_unless
from unittest import main
from eventlet import api, util, coros, greenio
import socket
import os
certificate_file = os.path.join(os.path.dirname(__file__), 'test_server.crt')
private_key_file = os.path.join(os.path.dirname(__file__), 'test_server.key')
class SSLTest(LimitedTestCase):
def test_duplex_response(self):
def serve(listener):
sock, addr = listener.accept()
stuff = sock.read(8192)
sock.write('response')
sock = api.ssl_listener(('127.0.0.1', 0), certificate_file, private_key_file)
server_coro = coros.execute(serve, sock)
client = util.wrap_ssl(api.connect_tcp(('127.0.0.1', sock.getsockname()[1])))
client.write('line 1\r\nline 2\r\n\r\n')
self.assertEquals(client.read(8192), 'response')
server_coro.wait()
def test_ssl_close(self):
def serve(listener):
sock, addr = listener.accept()
stuff = sock.read(8192)
try:
self.assertEquals("", sock.read(8192))
except greenio.SSL.ZeroReturnError:
pass
sock = api.ssl_listener(('127.0.0.1', 0), certificate_file, private_key_file)
server_coro = coros.execute(serve, sock)
raw_client = api.connect_tcp(('127.0.0.1', sock.getsockname()[1]))
client = util.wrap_ssl(raw_client)
client.write('X')
greenio.shutdown_safe(client)
client.close()
server_coro.wait()
class SocketSSLTest(LimitedTestCase):
@skip_unless(hasattr(socket, 'ssl'))
def test_greensslobject(self):
import warnings
# disabling socket.ssl warnings because we're testing it here
warnings.filterwarnings(action = 'ignore',
message='.*socket.ssl.*',
category=DeprecationWarning)
def serve(listener):
sock, addr = listener.accept()
sock.write('content')
greenio.shutdown_safe(sock)
sock.close()
listener = api.ssl_listener(('', 0),
certificate_file,
private_key_file)
killer = api.spawn(serve, listener)
from eventlet.green.socket import ssl
client = ssl(api.connect_tcp(('localhost', listener.getsockname()[1])))
self.assertEquals(client.read(1024), 'content')
self.assertEquals(client.read(1024), '')
if __name__ == '__main__':
main()