Thanks to Marcus for the SSL patch for 2.6 compatibility. Added greenlet and pyopenssl dependencies to setup.py.
This commit is contained in:
@@ -51,7 +51,7 @@ def build_opener(*handlers):
|
|||||||
default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
|
default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
|
||||||
HTTPDefaultErrorHandler, HTTPRedirectHandler,
|
HTTPDefaultErrorHandler, HTTPRedirectHandler,
|
||||||
FTPHandler, FileHandler, HTTPErrorProcessor]
|
FTPHandler, FileHandler, HTTPErrorProcessor]
|
||||||
if hasattr(httplib, 'HTTPS'):
|
if hasattr(urllib2, 'HTTPSHandler'):
|
||||||
default_classes.append(HTTPSHandler)
|
default_classes.append(HTTPSHandler)
|
||||||
skip = set()
|
skip = set()
|
||||||
for klass in default_classes:
|
for klass in default_classes:
|
||||||
|
@@ -59,7 +59,11 @@ def tcp_socket():
|
|||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
__original_ssl__ = socket.ssl
|
try:
|
||||||
|
import ssl
|
||||||
|
__original_ssl__ = ssl.wrap_socket
|
||||||
|
except ImportError:
|
||||||
|
__original_ssl__ = socket.ssl
|
||||||
except AttributeError:
|
except AttributeError:
|
||||||
__original_ssl__ = None
|
__original_ssl__ = None
|
||||||
|
|
||||||
|
52
examples/webcrawler.py
Normal file
52
examples/webcrawler.py
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
#! /usr/bin/env python
|
||||||
|
"""\
|
||||||
|
@file webcrawler.py
|
||||||
|
|
||||||
|
This is a simple web "crawler" that fetches a bunch of urls using a coroutine pool. It fetches as
|
||||||
|
many urls at time as coroutines in the pool.
|
||||||
|
|
||||||
|
Copyright (c) 2007, Linden Research, Inc.
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in
|
||||||
|
all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||||
|
THE SOFTWARE.
|
||||||
|
"""
|
||||||
|
|
||||||
|
urls = ["http://www.google.com/intl/en_ALL/images/logo.gif",
|
||||||
|
"http://us.i1.yimg.com/us.yimg.com/i/ww/beta/y3.gif",
|
||||||
|
"http://eventlet.net"]
|
||||||
|
|
||||||
|
import time
|
||||||
|
from eventlet.green import urllib2
|
||||||
|
from eventlet import coros
|
||||||
|
|
||||||
|
def fetch(url):
|
||||||
|
# we could do something interesting with the result, but this is
|
||||||
|
# example code, so we'll just report that we did it
|
||||||
|
print "%s fetching %s" % (time.asctime(), url)
|
||||||
|
req = urllib2.urlopen(url)
|
||||||
|
print "%s fetched %s (%s)" % (time.asctime(), url, len(req.read()))
|
||||||
|
|
||||||
|
pool = coros.CoroutinePool(max_size=4)
|
||||||
|
waiters = []
|
||||||
|
for url in urls:
|
||||||
|
waiters.append(pool.execute(fetch, url))
|
||||||
|
|
||||||
|
# wait for all the coroutines to come back before exiting the process
|
||||||
|
for waiter in waiters:
|
||||||
|
waiter.wait()
|
||||||
|
|
||||||
|
|
Reference in New Issue
Block a user