Refactoring example to use a bit clearer code flow.
This commit is contained in:
@@ -38,12 +38,17 @@ def producer(start_url):
|
|||||||
q = eventlet.Queue()
|
q = eventlet.Queue()
|
||||||
q.put(start_url)
|
q.put(start_url)
|
||||||
# keep looping if there are new urls, or workers that may produce more urls
|
# keep looping if there are new urls, or workers that may produce more urls
|
||||||
while not q.empty() or pool.running() != 0:
|
while True:
|
||||||
url = eventlet.with_timeout(0.1, q.get, timeout_value='')
|
while not q.empty():
|
||||||
|
url = q.get()
|
||||||
# limit requests to eventlet.net so we don't crash all over the internet
|
# limit requests to eventlet.net so we don't crash all over the internet
|
||||||
if url not in seen and 'eventlet.net' in url:
|
if url not in seen and 'eventlet.net' in url:
|
||||||
seen.add(url)
|
seen.add(url)
|
||||||
pool.spawn_n(fetch, url, q)
|
pool.spawn_n(fetch, url, q)
|
||||||
|
pool.waitall()
|
||||||
|
if q.empty():
|
||||||
|
break
|
||||||
|
|
||||||
return seen
|
return seen
|
||||||
|
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user