34 lines
953 B
Python
34 lines
953 B
Python
#! /usr/bin/env python
|
|
"""\
|
|
@file webcrawler.py
|
|
|
|
This is a simple web "crawler" that fetches a bunch of urls using a coroutine pool. It fetches as
|
|
many urls at time as coroutines in the pool.
|
|
"""
|
|
|
|
urls = ["http://www.google.com/intl/en_ALL/images/logo.gif",
|
|
"http://us.i1.yimg.com/us.yimg.com/i/ww/beta/y3.gif",
|
|
"http://eventlet.net"]
|
|
|
|
import time
|
|
from eventlet.green import urllib2
|
|
from eventlet import coros
|
|
|
|
def fetch(url):
|
|
# we could do something interesting with the result, but this is
|
|
# example code, so we'll just report that we did it
|
|
print "%s fetching %s" % (time.asctime(), url)
|
|
req = urllib2.urlopen(url)
|
|
print "%s fetched %s (%s)" % (time.asctime(), url, len(req.read()))
|
|
|
|
pool = coros.CoroutinePool(max_size=4)
|
|
waiters = []
|
|
for url in urls:
|
|
waiters.append(pool.execute(fetch, url))
|
|
|
|
# wait for all the coroutines to come back before exiting the process
|
|
for waiter in waiters:
|
|
waiter.wait()
|
|
|
|
|