Patcherized both urllib2 and its test.

eventlet/green/urllib2.py

@@ -1,253 +1,17 @@
-urllib2 = __import__('urllib2')
+from eventlet import patcher
-for var in dir(urllib2):
-    exec "%s = urllib2.%s" % (var, var)
-
-# import the following to be a better drop-in replacement
-__import_lst = ['__version__', '__cut_port_re', '_parse_proxy']
-
-for var in __import_lst:
-    exec "%s = getattr(urllib2, %r, None)" % (var, var)
-
-for x in ('urlopen', 'install_opener', 'build_opener', 'HTTPHandler', 'HTTPSHandler',
-          'HTTPCookieProcessor', 'FileHandler', 'FTPHandler', 'CacheFTPHandler', 'GopherError'):
-    globals().pop(x, None)
-
-from eventlet.green import httplib
-import mimetools
-import os
-from eventlet.green import socket
-import sys
-from eventlet.green import time
-
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
-
-from eventlet.green.urllib import (unwrap, unquote, splittype, splithost, quote,
-     addinfourl, splitport, splitquery,
-     splitattr, ftpwrapper, noheaders, splituser, splitpasswd, splitvalue)
-
-# support for FileHandler, proxies via environment variables
-from eventlet.green.urllib import localhost, url2pathname, getproxies
-
-_opener = None
-def urlopen(url, data=None):
-    global _opener
-    if _opener is None:
-        _opener = build_opener()
-    return _opener.open(url, data)
-
-def install_opener(opener):
-    global _opener
-    _opener = opener
-
-def build_opener(*handlers):
-    import types
-    def isclass(obj):
-        return isinstance(obj, types.ClassType) or hasattr(obj, "__bases__")
-
-    opener = OpenerDirector()
-    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
-                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
-                       FTPHandler, FileHandler, HTTPErrorProcessor]
-    if hasattr(urllib2, 'HTTPSHandler'):
-        default_classes.append(HTTPSHandler)
-    skip = set()
-    for klass in default_classes:
-        for check in handlers:
-            if isclass(check):
-                if issubclass(check, klass):
-                    skip.add(klass)
-            elif isinstance(check, klass):
-                skip.add(klass)
-    for klass in skip:
-        default_classes.remove(klass)
-
-    for klass in default_classes:
-        opener.add_handler(klass())
-
-    for h in handlers:
-        if isclass(h):
-            h = h()
-        opener.add_handler(h)
-    return opener
-
-class HTTPHandler(urllib2.HTTPHandler):
-
-    def http_open(self, req):
-        return self.do_open(httplib.HTTPConnection, req)
-
-    http_request = AbstractHTTPHandler.do_request_
-
-if hasattr(urllib2, 'HTTPSHandler'):
-    class HTTPSHandler(urllib2.HTTPSHandler):
-
-        def https_open(self, req):
-            return self.do_open(httplib.HTTPSConnection, req)
-
-        https_request = AbstractHTTPHandler.do_request_
-
-class HTTPCookieProcessor(urllib2.HTTPCookieProcessor):
-    def __init__(self, cookiejar=None):
-        from eventlet.green import cookielib
-        if cookiejar is None:
-            cookiejar = cookielib.CookieJar()
-        self.cookiejar = cookiejar
-
-class FileHandler(urllib2.FileHandler):
-
-    def get_names(self):
-        if FileHandler.names is None:
-            try:
-                FileHandler.names = (socket.gethostbyname('localhost'),
-                                    socket.gethostbyname(socket.gethostname()))
-            except socket.gaierror:
-                FileHandler.names = (socket.gethostbyname('localhost'),)
-        return FileHandler.names
-
-    def open_local_file(self, req):
-        import email.Utils
-        import mimetypes
-        host = req.get_host()
-        file = req.get_selector()
-        localfile = url2pathname(file)
-        stats = os.stat(localfile)
-        size = stats.st_size
-        modified = email.Utils.formatdate(stats.st_mtime, usegmt=True)
-        mtype = mimetypes.guess_type(file)[0]
-        headers = mimetools.Message(StringIO(
-            'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
-            (mtype or 'text/plain', size, modified)))
-        if host:
-            host, port = splitport(host)
-        if not host or \
-           (not port and socket.gethostbyname(host) in self.get_names()):
-            return addinfourl(open(localfile, 'rb'),
-                              headers, 'file:'+file)
-        raise URLError('file not on local host')
-
-class FTPHandler(urllib2.FTPHandler):
-    def ftp_open(self, req):
 from eventlet.green import ftplib
-        import mimetypes
+from eventlet.green import httplib
-        host = req.get_host()
+from eventlet.green import socket
-        if not host:
+from eventlet.green import time
-            raise IOError, ('ftp error', 'no host given')
+from eventlet.green import urllib
-        host, port = splitport(host)
-        if port is None:
-            port = ftplib.FTP_PORT
-        else:
-            port = int(port)
 
-        # username/password handling
+patcher.inject('urllib2',
-        user, host = splituser(host)
+    globals(),
-        if user:
+    ('httplib', httplib),
-            user, passwd = splitpasswd(user)
+    ('socket', socket),
-        else:
+    ('time', time),
-            passwd = None
+    ('urllib', urllib))
-        host = unquote(host)
-        user = unquote(user or '')
-        passwd = unquote(passwd or '')
 
-        try:
+FTPHandler.ftp_open = patcher.patch_function(FTPHandler.ftp_open, ('ftplib', ftplib))
-            host = socket.gethostbyname(host)
-        except socket.error, msg:
-            raise URLError(msg)
-        path, attrs = splitattr(req.get_selector())
-        dirs = path.split('/')
-        dirs = map(unquote, dirs)
-        dirs, file = dirs[:-1], dirs[-1]
-        if dirs and not dirs[0]:
-            dirs = dirs[1:]
-        try:
-            fw = self.connect_ftp(user, passwd, host, port, dirs)
-            type = file and 'I' or 'D'
-            for attr in attrs:
-                attr, value = splitvalue(attr)
-                if attr.lower() == 'type' and \
-                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
-                    type = value.upper()
-            fp, retrlen = fw.retrfile(file, type)
-            headers = ""
-            mtype = mimetypes.guess_type(req.get_full_url())[0]
-            if mtype:
-                headers += "Content-type: %s\n" % mtype
-            if retrlen is not None and retrlen >= 0:
-                headers += "Content-length: %d\n" % retrlen
-            sf = StringIO(headers)
-            headers = mimetools.Message(sf)
-            return addinfourl(fp, headers, req.get_full_url())
-        except ftplib.all_errors, msg:
-            raise IOError, ('ftp error', msg), sys.exc_info()[2]
-
-    def connect_ftp(self, user, passwd, host, port, dirs):
-        fw = ftpwrapper(user, passwd, host, port, dirs)
-##        fw.ftp.set_debuglevel(1)
-        return fw
-
-class CacheFTPHandler(FTPHandler):
-    # XXX would be nice to have pluggable cache strategies
-    # XXX this stuff is definitely not thread safe
-    def __init__(self):
-        self.cache = {}
-        self.timeout = {}
-        self.soonest = 0
-        self.delay = 60
-        self.max_conns = 16
-
-    def setTimeout(self, t):
-        self.delay = t
-
-    def setMaxConns(self, m):
-        self.max_conns = m
-
-    def connect_ftp(self, user, passwd, host, port, dirs):
-        key = user, host, port, '/'.join(dirs)
-        if key in self.cache:
-            self.timeout[key] = time.time() + self.delay
-        else:
-            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs)
-            self.timeout[key] = time.time() + self.delay
-        self.check_cache()
-        return self.cache[key]
-
-    def check_cache(self):
-        # first check for old ones
-        t = time.time()
-        if self.soonest <= t:
-            for k, v in self.timeout.items():
-                if v < t:
-                    self.cache[k].close()
-                    del self.cache[k]
-                    del self.timeout[k]
-        self.soonest = min(self.timeout.values())
-
-        # then check the size
-        if len(self.cache) == self.max_conns:
-            for k, v in self.timeout.items():
-                if v == self.soonest:
-                    del self.cache[k]
-                    del self.timeout[k]
-                    break
-            self.soonest = min(self.timeout.values())
-
-class GopherHandler(BaseHandler):
-    def gopher_open(self, req):
-        # XXX can raise socket.error
-        from eventlet.green import gopherlib  # this raises DeprecationWarning in 2.5
-        host = req.get_host()
-        if not host:
-            raise GopherError('no host given')
-        host = unquote(host)
-        selector = req.get_selector()
-        type, selector = splitgophertype(selector)
-        selector, query = splitquery(selector)
-        selector = unquote(selector)
-        if query:
-            query = unquote(query)
-            fp = gopherlib.send_query(selector, query, host)
-        else:
-            fp = gopherlib.send_selector(selector, host)
-        return addinfourl(fp, noheaders(), req.get_full_url())
 
+del patcher

tests/stdlib/test_urllib2.py

@@ -1,15 +1,13 @@
-from test import test_urllib2
+from eventlet import patcher
-
 from eventlet.green import socket
 from eventlet.green import urllib2
-from eventlet.green.urllib2 import Request, OpenerDirector
 
-test_urllib2.socket = socket
+patcher.inject('test.test_urllib2',
-test_urllib2.urllib2 = urllib2
+    globals(),
-test_urllib2.Request = Request
+    ('socket', socket),
-test_urllib2.OpenerDirector = OpenerDirector
+    ('urllib2', urllib2))
 
-from test.test_urllib2 import *
+HandlerTests.test_file = patcher.patch_function(HandlerTests.test_file, ('socket', socket))
 
 if __name__ == "__main__":
     test_main()