Updated NEWS, added GreenPool.starmap, fixed the annoying intermittent test failure.
This commit is contained in:
12
NEWS
12
NEWS
@@ -1,6 +1,18 @@
|
||||
0.9.3
|
||||
=====
|
||||
|
||||
* New debug module, used for enabling verbosity within Eventlet that can help debug applications or Eventlet itself.
|
||||
* Bugfixes in tpool, green.select
|
||||
* Moved primary api module to __init__ from api. It shouldn't be necessary to import eventlet.api anymore; import eventlet should do the same job.
|
||||
* Proc module deprecated in favor of greenthread
|
||||
* New module greenthread, with new class GreenThread.
|
||||
* New GreenPool class that replaces pool.Pool.
|
||||
* Deprecated coros.execute
|
||||
* Deprecated coros.semaphore
|
||||
* Moved coros.BoundedSemaphore to semaphore.BoundedSemaphore
|
||||
* Moved coros.Semaphore to semaphore.Semaphore
|
||||
* Moved coros.event to event.Event
|
||||
* Deprecated api.tcp_listener, api.connect_tcp, api.ssl_listener
|
||||
* Moved get_hub, use_hub, get_default_hub to eventlet.hubs
|
||||
* Renamed libevent hub to pyevent.
|
||||
* Renamed coros.event to coros.Event
|
||||
|
||||
@@ -125,25 +125,30 @@ class GreenPool(object):
|
||||
else:
|
||||
return 0
|
||||
|
||||
def _do_imap(self, func, it, gi):
|
||||
def _do_map(self, func, it, gi):
|
||||
for args in it:
|
||||
gi.spawn(func, *args)
|
||||
gi.spawn(raise_stop_iteration)
|
||||
|
||||
def imap(self, function, *iterables):
|
||||
"""This is the same as itertools.imap, except that *func* is
|
||||
executed in separate green threads, with the concurrency controlled by
|
||||
the pool. In operation, imap consumes a constant amount of memory,
|
||||
proportional to the size of the pool, and is thus suited for iterating
|
||||
over extremely long input lists.
|
||||
|
||||
def starmap(self, function, iterable):
|
||||
"""This is the same as :func:`itertools.starmap`, except that *func* is
|
||||
executed in a separate green thread for each item, with the concurrency
|
||||
limited by the pool's size. In operation, starmap consumes a constant
|
||||
amount of memory, proportional to the size of the pool, and is thus
|
||||
suited for iterating over extremely long input lists.
|
||||
"""
|
||||
if function is None:
|
||||
function = lambda *a: a
|
||||
it = itertools.izip(*iterables)
|
||||
gi = GreenImap(self.size)
|
||||
greenthread.spawn_n(self._do_imap, function, it, gi)
|
||||
gi = GreenMap(self.size)
|
||||
greenthread.spawn_n(self._do_map, function, iterable, gi)
|
||||
return gi
|
||||
|
||||
def imap(self, function, *iterables):
|
||||
"""This is the same as :func:`itertools.imap`, and has the same
|
||||
concurrency and memory behavior as :meth:`starmap`.
|
||||
"""
|
||||
return self.starmap(function, itertools.izip(*iterables))
|
||||
|
||||
|
||||
def raise_stop_iteration():
|
||||
raise StopIteration()
|
||||
@@ -158,7 +163,11 @@ class GreenPile(object):
|
||||
|
||||
A GreenPile can also be constructed standalone, not associated with any
|
||||
GreenPool. To do this, construct it with an integer size parameter instead
|
||||
of a GreenPool
|
||||
of a GreenPool.
|
||||
|
||||
It is not advisable to iterate over a GreenPile in a different greenthread
|
||||
than the one which is calling spawn. The iterator will exit early in that
|
||||
situation.
|
||||
"""
|
||||
def __init__(self, size_or_pool=1000):
|
||||
if isinstance(size_or_pool, GreenPool):
|
||||
@@ -195,8 +204,15 @@ class GreenPile(object):
|
||||
self.counter -= 1
|
||||
|
||||
# this is identical to GreenPile but it blocks on spawn if the results
|
||||
# aren't consumed
|
||||
class GreenImap(GreenPile):
|
||||
# aren't consumed, and it doesn't generate its own StopIteration exception,
|
||||
# instead relying on the spawning process to send one in when it's done
|
||||
class GreenMap(GreenPile):
|
||||
def __init__(self, size_or_pool):
|
||||
super(GreenImap, self).__init__(size_or_pool)
|
||||
self.waiters = coros.Channel(max_size=self.pool.size)
|
||||
super(GreenMap, self).__init__(size_or_pool)
|
||||
self.waiters = coros.Channel(max_size=self.pool.size)
|
||||
|
||||
def next(self):
|
||||
try:
|
||||
return self.waiters.wait().wait()
|
||||
finally:
|
||||
self.counter -= 1
|
||||
@@ -275,6 +275,10 @@ class GreenPool(tests.LimitedTestCase):
|
||||
break
|
||||
self.assertEquals(results, [0,'r',2,3,4,5,6,'r',8,9])
|
||||
|
||||
def test_starmap(self):
|
||||
p = greenpool.GreenPool(4)
|
||||
result_list = list(p.starmap(passthru, [(x,) for x in xrange(10)]))
|
||||
self.assertEquals(result_list, range(10))
|
||||
|
||||
class GreenPile(tests.LimitedTestCase):
|
||||
def test_pile(self):
|
||||
|
||||
Reference in New Issue
Block a user