Thanks to Marcus for the SSL patch for 2.6 compatibility. Added greenlet and pyopenssl dependencies to setup.py.
This commit is contained in:
@@ -51,7 +51,7 @@ def build_opener(*handlers):
|
||||
default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
|
||||
HTTPDefaultErrorHandler, HTTPRedirectHandler,
|
||||
FTPHandler, FileHandler, HTTPErrorProcessor]
|
||||
if hasattr(httplib, 'HTTPS'):
|
||||
if hasattr(urllib2, 'HTTPSHandler'):
|
||||
default_classes.append(HTTPSHandler)
|
||||
skip = set()
|
||||
for klass in default_classes:
|
||||
|
@@ -59,6 +59,10 @@ def tcp_socket():
|
||||
|
||||
|
||||
try:
|
||||
try:
|
||||
import ssl
|
||||
__original_ssl__ = ssl.wrap_socket
|
||||
except ImportError:
|
||||
__original_ssl__ = socket.ssl
|
||||
except AttributeError:
|
||||
__original_ssl__ = None
|
||||
|
52
examples/webcrawler.py
Normal file
52
examples/webcrawler.py
Normal file
@@ -0,0 +1,52 @@
|
||||
#! /usr/bin/env python
|
||||
"""\
|
||||
@file webcrawler.py
|
||||
|
||||
This is a simple web "crawler" that fetches a bunch of urls using a coroutine pool. It fetches as
|
||||
many urls at time as coroutines in the pool.
|
||||
|
||||
Copyright (c) 2007, Linden Research, Inc.
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
"""
|
||||
|
||||
urls = ["http://www.google.com/intl/en_ALL/images/logo.gif",
|
||||
"http://us.i1.yimg.com/us.yimg.com/i/ww/beta/y3.gif",
|
||||
"http://eventlet.net"]
|
||||
|
||||
import time
|
||||
from eventlet.green import urllib2
|
||||
from eventlet import coros
|
||||
|
||||
def fetch(url):
|
||||
# we could do something interesting with the result, but this is
|
||||
# example code, so we'll just report that we did it
|
||||
print "%s fetching %s" % (time.asctime(), url)
|
||||
req = urllib2.urlopen(url)
|
||||
print "%s fetched %s (%s)" % (time.asctime(), url, len(req.read()))
|
||||
|
||||
pool = coros.CoroutinePool(max_size=4)
|
||||
waiters = []
|
||||
for url in urls:
|
||||
waiters.append(pool.execute(fetch, url))
|
||||
|
||||
# wait for all the coroutines to come back before exiting the process
|
||||
for waiter in waiters:
|
||||
waiter.wait()
|
||||
|
||||
|
Reference in New Issue
Block a user