Documented the environment variables (thereby fixing #15). Improved the implementation of tpooled dns in eventlet.green.socket.

doc/Makefile

@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = PYTHONPATH=../:$(PYTHONPATH) sphinx-build
+SPHINXBUILD   = PYTHONPATH=../:$(PYTHONPATH) ~/p/bin/sphinx-build
 PAPER         =
 
 # Internal variables.

doc/environment.rst

@@ -0,0 +1,26 @@
+Environment Variables
+======================
+
+Eventlet's behavior can be controlled by a few environment variables.
+These are only for the advanced user.
+
+EVENTLET_HUB 
+
+   Used to force Eventlet to use the specified hub instead of the
+   optimal one.  See :ref:`understanding_hubs` for the list of
+   acceptable hubs and what they mean (note that picking a hub not on
+   the list will silently fail).  Equivalent to calling
+   :meth:`eventlet.hubs.use_hub` at the beginning of the program.
+
+EVENTLET_THREADPOOL_SIZE
+
+   The size of the threadpool in :mod:`~eventlet.tpool`.  This is an
+   environment variable because tpool constructs its pool on first
+   use, so any control of the pool size needs to happen before then.
+
+EVENTLET_TPOOL_DNS 
+
+   If set to 'yes', uses :func:`eventlet.tpool.execute` to call
+   :func:`~socket.gethostbyname` and :func:`~socket.getaddrinfo`,
+   making them appear non-blocking.  This environment variable is
+   ignored on OS X.

doc/index.rst

@@ -31,6 +31,7 @@ Contents
    threading
    hubs
    testing
+   environment
 
    modules
    

eventlet/green/socket.py

@@ -12,8 +12,8 @@ os = __import__('os')
 import sys
 import warnings
 
-__patched__ = ['fromfd', 'socketpair', 'gethostbyname', 'create_connection',
-               'ssl', 'socket']
+__patched__ = ['fromfd', 'socketpair', 'gethostbyname', 'getaddrinfo',
+               'create_connection', 'ssl', 'socket']
 
 try:
     __original_fromfd__ = __socket.fromfd
@@ -31,20 +31,11 @@ except AttributeError:
     pass
 
 __original_gethostbyname__ = __socket.gethostbyname
-def gethostbyname(name):
-    can_use_tpool = os.environ.get("EVENTLET_TPOOL_GETHOSTBYNAME",
-                                   '').lower() == "yes"
-    if getattr(get_hub(), 'uses_twisted_reactor', None):
-        globals()['gethostbyname'] = _gethostbyname_twisted
-    elif sys.platform.startswith('darwin') or not can_use_tpool:
-        # the thread primitives on Darwin have some bugs that make
-        # it undesirable to use tpool for hostname lookups
-        globals()['gethostbyname'] = __original_gethostbyname__
-    else:
-        globals()['gethostbyname'] = _gethostbyname_tpool
-
-    return globals()['gethostbyname'](name)
-
+# the thread primitives on Darwin have some bugs that make
+# it undesirable to use tpool for hostname lookups
+_can_use_tpool = (
+    os.environ.get("EVENTLET_TPOOL_DNS",'').lower() == "yes"
+    and not sys.platform.startswith('darwin'))
 def _gethostbyname_twisted(name):
     from twisted.internet import reactor
     from eventlet.twistedutil import block_on as _block_on
@@ -55,12 +46,25 @@ def _gethostbyname_tpool(name):
     return tpool.execute(
         __original_gethostbyname__, name)
 
-#     def getaddrinfo(*args, **kw):
-#         return tpool.execute(
-#             __socket.getaddrinfo, *args, **kw)
-#
-# XXX there're few more blocking functions in socket
-# XXX having a hub-independent way to access thread pool would be nice
+if getattr(get_hub(), 'uses_twisted_reactor', None):
+    gethostbyname = _gethostbyname_twisted
+elif _can_use_tpool:
+    gethostbyname = _gethostbyname_tpool
+else:
+    gethostbyname = __original_gethostbyname__
+
+
+__original_getaddrinfo__ = __socket.getaddrinfo
+def _getaddrinfo_tpool(*args, **kw):
+    from eventlet import tpool
+    return tpool.execute(
+        __original_getaddrinfo__, *args, **kw)
+
+if _can_use_tpool:
+    getaddrinfo = _getaddrinfo_tpool
+else:
+    getaddrinfo = __original_getaddrinfo__
+
 
 def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT):
     """Connect to *address* and return the socket object.

tests/env_test.py

@@ -0,0 +1,55 @@
+import os
+from tests.patcher_test import ProcessBase
+
+class Socket(ProcessBase):
+    def test_patched_thread(self):
+        new_mod = """from eventlet.green import socket
+socket.gethostbyname('localhost')
+socket.getaddrinfo('localhost', 80)
+"""
+        os.environ['EVENTLET_TPOOL_DNS'] = 'yes'
+        try:
+            self.write_to_tempfile("newmod", new_mod)
+            output, lines = self.launch_subprocess('newmod.py')
+            self.assertEqual(len(lines), 1, lines)
+        finally:
+            del os.environ['EVENTLET_TPOOL_DNS']
+    
+    def test_tpool_size(self):
+        new_mod = """from eventlet import tpool
+import eventlet
+import time
+current = [0]
+highwater = [0]
+def count():
+    current[0] += 1
+    time.sleep(0.01)
+    if current[0] > highwater[0]:
+        highwater[0] = current[0]
+    current[0] -= 1
+expected = 40
+p = eventlet.GreenPool()
+for i in xrange(expected):
+    p.spawn(tpool.execute,count)
+p.waitall()
+assert highwater[0] == expected, "%s != %s" % (highwater[0], expected)"""
+        os.environ['EVENTLET_THREADPOOL_SIZE'] = "40"
+        try:
+            self.write_to_tempfile("newmod", new_mod)
+            output, lines = self.launch_subprocess('newmod.py')
+            self.assertEqual(len(lines), 1, lines)
+        finally:
+            del os.environ['EVENTLET_THREADPOOL_SIZE']
+
+    def test_eventlet_hub(self):
+        new_mod = """from eventlet import hubs
+print hubs.get_hub()
+"""
+        os.environ['EVENTLET_HUB'] = 'selects'
+        try:
+            self.write_to_tempfile("newmod", new_mod)
+            output, lines = self.launch_subprocess('newmod.py')
+            self.assertEqual(len(lines), 2, "\n".join(lines))
+            self.assert_("selects" in lines[0])
+        finally:
+            del os.environ['EVENTLET_HUB']

tests/patcher_test.py

@@ -27,7 +27,7 @@ import socket
 print "importing", patching, socket, patching.socket, patching.urllib
 """
 
-class Patcher(LimitedTestCase):
+class ProcessBase(LimitedTestCase):
     TEST_TIMEOUT=3 # starting processes is time-consuming
     def setUp(self):
         self._saved_syspath = sys.path
@@ -55,7 +55,7 @@ class Patcher(LimitedTestCase):
         return output, lines
 
 
-class ImportPatched(Patcher):
+class ImportPatched(ProcessBase):
     def test_patch_a_module(self):
         self.write_to_tempfile("base", base_module_contents)
         self.write_to_tempfile("patching", patching_module_contents)
@@ -85,7 +85,7 @@ print "newmod", base, base.socket, base.urllib.socket.socket
         self.assert_('GreenSocket' in lines[1], repr(output))
 
 
-class MonkeyPatch(Patcher):        
+class MonkeyPatch(ProcessBase):        
     def test_patched_modules(self):
         new_mod = """
 from eventlet import patcher
@@ -220,7 +220,7 @@ def test_monkey_patch_threading():
     assert tickcount[0] > 900
 """
 
-class Tpool(Patcher):
+class Tpool(ProcessBase):
     TEST_TIMEOUT=3
 
     @skip_with_pyevent