Merge pull request #38 from cdent/add-urllib3

Add support for intercepting urllib3

docs/index.rst

@@ -15,6 +15,7 @@ Examples
    http_client
    httplib2
    requests
+   urllib3
    urllib
 
 

docs/urllib3.rst

@@ -0,0 +1,32 @@
+urllib3_intercept
+==================
+
+.. automodule:: wsgi_intercept.urllib3_intercept
+
+
+Example:
+
+.. testcode:: 
+
+    import urllib3
+    from wsgi_intercept import urllib3_intercept, add_wsgi_intercept
+
+    pool = urllib3.PoolManager()
+
+
+    def app(environ, start_response):
+        start_response('200 OK', [('Content-Type', 'text/plain')])
+        return [b'Whee']
+
+
+    def make_app():
+        return app
+
+
+    host, port = 'localhost', 80
+    url = 'http://{0}:{1}/'.format(host, port)
+    urllib3_intercept.install()
+    add_wsgi_intercept(host, port, make_app)
+    resp = pool.requests('GET', url)
+    assert resp.data == b'Whee'
+    urllib3_intercept.uninstall()

setup.py

@@ -38,6 +38,7 @@ META = {
             'pytest>=2.4',
             'httplib2',
             'requests>=2.0.1',
+            'urllib3',
         ],
     },
 }

test/test_interceptor.py

@@ -9,6 +9,7 @@ from uuid import uuid4
 
 import py.test
 import requests
+import urllib3
 from httplib2 import Http, ServerNotFoundError
 from six.moves import http_client
 from six.moves.urllib.request import urlopen
@@ -16,9 +17,10 @@ from six.moves.urllib.error import URLError
 
 from wsgi_intercept.interceptor import (
     Interceptor, HttpClientInterceptor, Httplib2Interceptor,
-    RequestsInterceptor, UrllibInterceptor)
+    RequestsInterceptor, UrllibInterceptor, Urllib3Interceptor)
 from .wsgi_app import simple_app
 
+httppool = urllib3.PoolManager()
 
 def app():
     return simple_app
@@ -178,6 +180,41 @@ def test_requests_in_out():
         requests.get(url)
 
 
+# urllib3
+
+def test_urllib3_interceptor_host():
+    hostname = str(uuid4())
+    port = 9999
+    with Urllib3Interceptor(app=app, host=hostname, port=port) as url:
+        response = httppool.request('GET', url)
+        assert response.status == 200
+        assert 'WSGI intercept successful!' in str(response.data)
+
+
+def test_urllib3_interceptor_url():
+    hostname = str(uuid4())
+    port = 9999
+    url = 'http://%s:%s/' % (hostname, port)
+    with Urllib3Interceptor(app=app, url=url) as target_url:
+        response = httppool.request('GET', target_url)
+        assert response.status == 200
+        assert 'WSGI intercept successful!' in str(response.data)
+
+
+def test_urllib3_in_out():
+    hostname = str(uuid4())
+    port = 9999
+    url = 'http://%s:%s/' % (hostname, port)
+    with Urllib3Interceptor(app=app, url=url) as target_url:
+        response = httppool.request('GET', target_url)
+        assert response.status == 200
+        assert 'WSGI intercept successful!' in str(response.data)
+
+    # outside the context manager the intercept does not work
+    with py.test.raises(urllib3.exceptions.MaxRetryError):
+        httppool.request('GET', url)
+
+
 # urllib
 
 def test_urllib_interceptor_host():

test/test_urllib3.py

@@ -0,0 +1,90 @@
+import os
+import py.test
+import socket
+from wsgi_intercept import urllib3_intercept, WSGIAppError
+from test import wsgi_app
+from test.install import installer_class, skipnetwork
+import urllib3
+
+HOST = 'some_hopefully_nonexistant_domain'
+
+InstalledApp = installer_class(urllib3_intercept)
+http = urllib3.PoolManager()
+
+
+def test_http():
+    with InstalledApp(wsgi_app.simple_app, host=HOST, port=80) as app:
+        resp = http.request('GET', 'http://some_hopefully_nonexistant_domain:80/')
+        assert resp.data == b'WSGI intercept successful!\n'
+        assert app.success()
+
+
+def test_http_default_port():
+    with InstalledApp(wsgi_app.simple_app, host=HOST, port=80) as app:
+        resp = http.request('GET', 'http://some_hopefully_nonexistant_domain/')
+        assert resp.data == b'WSGI intercept successful!\n'
+        assert app.success()
+
+
+def test_http_other_port():
+    with InstalledApp(wsgi_app.simple_app, host=HOST, port=8080) as app:
+        resp = http.request('GET', 'http://some_hopefully_nonexistant_domain:8080/')
+        assert resp.data == b'WSGI intercept successful!\n'
+        assert app.success()
+        environ = app.get_internals()
+        assert environ['wsgi.url_scheme'] == 'http'
+
+
+def test_bogus_domain():
+    with InstalledApp(wsgi_app.simple_app, host=HOST, port=80):
+        py.test.raises(
+            urllib3.exceptions.MaxRetryError,
+            'http.request("GET", "http://_nonexistant_domain_")')
+
+
+def test_proxy_handling():
+    with py.test.raises(RuntimeError) as exc:
+        with InstalledApp(wsgi_app.simple_app, host=HOST, port=80,
+                          proxy='some_proxy.com:1234'):
+            http.request('GET', 'http://some_hopefully_nonexistant_domain:80/')
+    assert 'http_proxy or https_proxy set in environment' in str(exc.value)
+    # We need to do this by hand because the exception was raised
+    # during the entry of the context manager, so the exit handler
+    # wasn't reached.
+    del os.environ['http_proxy']
+
+
+def test_https():
+    with InstalledApp(wsgi_app.simple_app, host=HOST, port=443) as app:
+        resp = http.request('GET', 'https://some_hopefully_nonexistant_domain:443/')
+        assert resp.data == b'WSGI intercept successful!\n'
+        assert app.success()
+
+
+def test_https_default_port():
+    with InstalledApp(wsgi_app.simple_app, host=HOST, port=443) as app:
+        resp = http.request('GET', 'https://some_hopefully_nonexistant_domain/')
+        assert resp.data == b'WSGI intercept successful!\n'
+        assert app.success()
+        environ = app.get_internals()
+        assert environ['wsgi.url_scheme'] == 'https'
+
+
+def test_app_error():
+    with InstalledApp(wsgi_app.raises_app, host=HOST, port=80):
+        with py.test.raises(WSGIAppError):
+            http.request('GET', 'http://some_hopefully_nonexistant_domain/')
+
+
+@skipnetwork
+def test_http_not_intercepted():
+    with InstalledApp(wsgi_app.raises_app, host=HOST, port=80):
+        resp = http.request('GET', 'http://google.com')
+        assert resp.status >= 200 and resp.status < 300
+
+
+@skipnetwork
+def test_https_not_intercepted():
+    with InstalledApp(wsgi_app.raises_app, host=HOST, port=80):
+        resp = http.request('GET', 'https://google.com')
+        assert resp.status >= 200 and resp.status < 300

wsgi_intercept/_urllib3.py

@@ -0,0 +1,45 @@
+"""Common code of urllib3 and requests intercepts."""
+
+import os
+import sys
+
+from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket
+
+
+wsgi_fake_socket.settimeout = lambda self, timeout: None
+
+
+def make_urllib3_override(HTTPConnectionPool, HTTPSConnectionPool,
+                          HTTPConnection, HTTPSConnection):
+
+    class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection):
+        def __init__(self, *args, **kwargs):
+            if 'strict' in kwargs and sys.version_info > (3, 0):
+                kwargs.pop('strict')
+            WSGI_HTTPConnection.__init__(self, *args, **kwargs)
+            HTTPConnection.__init__(self, *args, **kwargs)
+
+
+    class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection):
+        is_verified = True
+
+        def __init__(self, *args, **kwargs):
+            if 'strict' in kwargs and sys.version_info > (3, 0):
+                kwargs.pop('strict')
+            WSGI_HTTPSConnection.__init__(self, *args, **kwargs)
+            HTTPSConnection.__init__(self, *args, **kwargs)
+
+
+    def install():
+        if 'http_proxy' in os.environ or 'https_proxy' in os.environ:
+            raise RuntimeError(
+                'http_proxy or https_proxy set in environment, please unset')
+        HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor
+        HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor
+
+
+    def uninstall():
+        HTTPConnectionPool.ConnectionCls = HTTPConnection
+        HTTPSConnectionPool.ConnectionCls = HTTPSConnection
+
+    return install, uninstall

wsgi_intercept/interceptor.py

@@ -109,6 +109,12 @@ class RequestsInterceptor(Interceptor):
     MODULE_NAME = 'requests_intercept'
 
 
+class Urllib3Interceptor(Interceptor):
+    """Interceptor for requests."""
+
+    MODULE_NAME = 'urllib3_intercept'
+
+
 class UrllibInterceptor(Interceptor):
     """Interceptor for urllib2 and urllib.request."""
 

wsgi_intercept/requests_intercept.py

@@ -2,45 +2,14 @@
 `requests <http://docs.python-requests.org/en/latest/>`_.
 """
 
-import os
-import sys
-
-from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket
 from requests.packages.urllib3.connectionpool import (HTTPConnectionPool,
         HTTPSConnectionPool)
 from requests.packages.urllib3.connection import (HTTPConnection,
         HTTPSConnection)
+from ._urllib3 import make_urllib3_override
 
 
-wsgi_fake_socket.settimeout = lambda self, timeout: None
+install, uninstall = make_urllib3_override(HTTPConnectionPool,
-
+                                           HTTPSConnectionPool,
-
+                                           HTTPConnection,
-class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection):
+                                           HTTPSConnection)
-    def __init__(self, *args, **kwargs):
-        if 'strict' in kwargs and sys.version_info > (3, 0):
-            kwargs.pop('strict')
-        WSGI_HTTPConnection.__init__(self, *args, **kwargs)
-        HTTPConnection.__init__(self, *args, **kwargs)
-
-
-class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection):
-    is_verified = True
-
-    def __init__(self, *args, **kwargs):
-        if 'strict' in kwargs and sys.version_info > (3, 0):
-            kwargs.pop('strict')
-        WSGI_HTTPSConnection.__init__(self, *args, **kwargs)
-        HTTPSConnection.__init__(self, *args, **kwargs)
-
-
-def install():
-    if 'http_proxy' in os.environ or 'https_proxy' in os.environ:
-        raise RuntimeError(
-            'http_proxy or https_proxy set in environment, please unset')
-    HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor
-    HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor
-
-
-def uninstall():
-    HTTPConnectionPool.ConnectionCls = HTTPConnection
-    HTTPSConnectionPool.ConnectionCls = HTTPSConnection

wsgi_intercept/urllib3_intercept.py

@@ -0,0 +1,13 @@
+"""Intercept HTTP connections that use
+`urllib3 <https://urllib3.readthedocs.org/>`_.
+"""
+
+from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
+from urllib3.connection import HTTPConnection, HTTPSConnection
+from ._urllib3 import make_urllib3_override
+
+
+install, uninstall = make_urllib3_override(HTTPConnectionPool,
+                                           HTTPSConnectionPool,
+                                           HTTPConnection,
+                                           HTTPSConnection)