diff --git a/doc/Makefile b/doc/Makefile index 076db3a..ae5567e 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line. SPHINXOPTS = -SPHINXBUILD = PYTHONPATH=../:$(PYTHONPATH) sphinx-build +SPHINXBUILD = PYTHONPATH=../:$(PYTHONPATH) ~/p/bin/sphinx-build PAPER = # Internal variables. diff --git a/doc/environment.rst b/doc/environment.rst new file mode 100644 index 0000000..29387e6 --- /dev/null +++ b/doc/environment.rst @@ -0,0 +1,26 @@ +Environment Variables +====================== + +Eventlet's behavior can be controlled by a few environment variables. +These are only for the advanced user. + +EVENTLET_HUB + + Used to force Eventlet to use the specified hub instead of the + optimal one. See :ref:`understanding_hubs` for the list of + acceptable hubs and what they mean (note that picking a hub not on + the list will silently fail). Equivalent to calling + :meth:`eventlet.hubs.use_hub` at the beginning of the program. + +EVENTLET_THREADPOOL_SIZE + + The size of the threadpool in :mod:`~eventlet.tpool`. This is an + environment variable because tpool constructs its pool on first + use, so any control of the pool size needs to happen before then. + +EVENTLET_TPOOL_DNS + + If set to 'yes', uses :func:`eventlet.tpool.execute` to call + :func:`~socket.gethostbyname` and :func:`~socket.getaddrinfo`, + making them appear non-blocking. This environment variable is + ignored on OS X. diff --git a/doc/index.rst b/doc/index.rst index 581e94b..2682edc 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -31,6 +31,7 @@ Contents threading hubs testing + environment modules diff --git a/eventlet/green/socket.py b/eventlet/green/socket.py index 75fe442..866c5ea 100644 --- a/eventlet/green/socket.py +++ b/eventlet/green/socket.py @@ -12,8 +12,8 @@ os = __import__('os') import sys import warnings -__patched__ = ['fromfd', 'socketpair', 'gethostbyname', 'create_connection', - 'ssl', 'socket'] +__patched__ = ['fromfd', 'socketpair', 'gethostbyname', 'getaddrinfo', + 'create_connection', 'ssl', 'socket'] try: __original_fromfd__ = __socket.fromfd @@ -31,20 +31,11 @@ except AttributeError: pass __original_gethostbyname__ = __socket.gethostbyname -def gethostbyname(name): - can_use_tpool = os.environ.get("EVENTLET_TPOOL_GETHOSTBYNAME", - '').lower() == "yes" - if getattr(get_hub(), 'uses_twisted_reactor', None): - globals()['gethostbyname'] = _gethostbyname_twisted - elif sys.platform.startswith('darwin') or not can_use_tpool: - # the thread primitives on Darwin have some bugs that make - # it undesirable to use tpool for hostname lookups - globals()['gethostbyname'] = __original_gethostbyname__ - else: - globals()['gethostbyname'] = _gethostbyname_tpool - - return globals()['gethostbyname'](name) - +# the thread primitives on Darwin have some bugs that make +# it undesirable to use tpool for hostname lookups +_can_use_tpool = ( + os.environ.get("EVENTLET_TPOOL_DNS",'').lower() == "yes" + and not sys.platform.startswith('darwin')) def _gethostbyname_twisted(name): from twisted.internet import reactor from eventlet.twistedutil import block_on as _block_on @@ -55,12 +46,25 @@ def _gethostbyname_tpool(name): return tpool.execute( __original_gethostbyname__, name) -# def getaddrinfo(*args, **kw): -# return tpool.execute( -# __socket.getaddrinfo, *args, **kw) -# -# XXX there're few more blocking functions in socket -# XXX having a hub-independent way to access thread pool would be nice +if getattr(get_hub(), 'uses_twisted_reactor', None): + gethostbyname = _gethostbyname_twisted +elif _can_use_tpool: + gethostbyname = _gethostbyname_tpool +else: + gethostbyname = __original_gethostbyname__ + + +__original_getaddrinfo__ = __socket.getaddrinfo +def _getaddrinfo_tpool(*args, **kw): + from eventlet import tpool + return tpool.execute( + __original_getaddrinfo__, *args, **kw) + +if _can_use_tpool: + getaddrinfo = _getaddrinfo_tpool +else: + getaddrinfo = __original_getaddrinfo__ + def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT): """Connect to *address* and return the socket object. diff --git a/tests/env_test.py b/tests/env_test.py new file mode 100644 index 0000000..fdd0b2d --- /dev/null +++ b/tests/env_test.py @@ -0,0 +1,55 @@ +import os +from tests.patcher_test import ProcessBase + +class Socket(ProcessBase): + def test_patched_thread(self): + new_mod = """from eventlet.green import socket +socket.gethostbyname('localhost') +socket.getaddrinfo('localhost', 80) +""" + os.environ['EVENTLET_TPOOL_DNS'] = 'yes' + try: + self.write_to_tempfile("newmod", new_mod) + output, lines = self.launch_subprocess('newmod.py') + self.assertEqual(len(lines), 1, lines) + finally: + del os.environ['EVENTLET_TPOOL_DNS'] + + def test_tpool_size(self): + new_mod = """from eventlet import tpool +import eventlet +import time +current = [0] +highwater = [0] +def count(): + current[0] += 1 + time.sleep(0.01) + if current[0] > highwater[0]: + highwater[0] = current[0] + current[0] -= 1 +expected = 40 +p = eventlet.GreenPool() +for i in xrange(expected): + p.spawn(tpool.execute,count) +p.waitall() +assert highwater[0] == expected, "%s != %s" % (highwater[0], expected)""" + os.environ['EVENTLET_THREADPOOL_SIZE'] = "40" + try: + self.write_to_tempfile("newmod", new_mod) + output, lines = self.launch_subprocess('newmod.py') + self.assertEqual(len(lines), 1, lines) + finally: + del os.environ['EVENTLET_THREADPOOL_SIZE'] + + def test_eventlet_hub(self): + new_mod = """from eventlet import hubs +print hubs.get_hub() +""" + os.environ['EVENTLET_HUB'] = 'selects' + try: + self.write_to_tempfile("newmod", new_mod) + output, lines = self.launch_subprocess('newmod.py') + self.assertEqual(len(lines), 2, "\n".join(lines)) + self.assert_("selects" in lines[0]) + finally: + del os.environ['EVENTLET_HUB'] diff --git a/tests/patcher_test.py b/tests/patcher_test.py index 3bc49db..500762d 100644 --- a/tests/patcher_test.py +++ b/tests/patcher_test.py @@ -27,7 +27,7 @@ import socket print "importing", patching, socket, patching.socket, patching.urllib """ -class Patcher(LimitedTestCase): +class ProcessBase(LimitedTestCase): TEST_TIMEOUT=3 # starting processes is time-consuming def setUp(self): self._saved_syspath = sys.path @@ -55,7 +55,7 @@ class Patcher(LimitedTestCase): return output, lines -class ImportPatched(Patcher): +class ImportPatched(ProcessBase): def test_patch_a_module(self): self.write_to_tempfile("base", base_module_contents) self.write_to_tempfile("patching", patching_module_contents) @@ -85,7 +85,7 @@ print "newmod", base, base.socket, base.urllib.socket.socket self.assert_('GreenSocket' in lines[1], repr(output)) -class MonkeyPatch(Patcher): +class MonkeyPatch(ProcessBase): def test_patched_modules(self): new_mod = """ from eventlet import patcher @@ -220,7 +220,7 @@ def test_monkey_patch_threading(): assert tickcount[0] > 900 """ -class Tpool(Patcher): +class Tpool(ProcessBase): TEST_TIMEOUT=3 @skip_with_pyevent