Browse Source

Improve get optimal count of max_worker for pool

In the upstream of futures library, the DEFAULT value max_worker
for ThreadPoolExecutor and ProcessPoolExecutor has been changed to:

ThreadPoolExecutor

max_worker = cpu_count() * 5

Reference:
https://github.com/agronholm/pythonfutures/blob/master/concurrent/futures/thread.py#L109

ProcessPoolExecutor

max_worker = cpu_count()

Reference:
https://github.com/agronholm/pythonfutures/blob/master/concurrent/futures/process.py#L278

So we should do it as well.

Change-Id: Id68e38a639c41426f125180ad335fb1fb97cba59
Closes-bug: #1739034
Chenjun Shen 1 year ago
parent
commit
8fff2242ff

+ 1
- 1
futurist/_futures.py View File

@@ -201,7 +201,7 @@ class ProcessPoolExecutor(_process.ProcessPoolExecutor):
201 201
 
202 202
     def __init__(self, max_workers=None):
203 203
         if max_workers is None:
204
-            max_workers = _utils.get_optimal_thread_count()
204
+            max_workers = _utils.get_optimal_process_count()
205 205
         super(ProcessPoolExecutor, self).__init__(max_workers=max_workers)
206 206
         if self._max_workers <= 0:
207 207
             raise ValueError("Max workers must be greater than zero")

+ 10
- 5
futurist/_utils.py View File

@@ -128,14 +128,19 @@ def get_callback_name(cb):
128 128
         return ".".join(segments)
129 129
 
130 130
 
131
-def get_optimal_thread_count(default=2):
131
+def get_optimal_thread_count(default=5):
132 132
     """Try to guess optimal thread count for current system."""
133 133
     try:
134
-        return multiprocessing.cpu_count() + 1
134
+        return multiprocessing.cpu_count() * 5
135
+    except NotImplementedError:
136
+        return default
137
+
138
+
139
+def get_optimal_process_count(default=1):
140
+    """Try to guess optimal process count for current system."""
141
+    try:
142
+        return multiprocessing.cpu_count()
135 143
     except NotImplementedError:
136
-        # NOTE(harlowja): apparently may raise so in this case we will
137
-        # just setup two threads since it's hard to know what else we
138
-        # should do in this situation.
139 144
         return default
140 145
 
141 146
 

+ 8
- 0
releasenotes/notes/improve-get-optimal-count-of-max_workers-for-pool-89368859d3b819e0.yaml View File

@@ -0,0 +1,8 @@
1
+---
2
+features:
3
+  - Improve get_optimal_thread_count() function for
4
+    choosing a new default value which is 5 * cpu_count()
5
+    for max_workers used by ThreadPoolExecutor.
6
+  - Add a new get_optimal_process_count() function
7
+    for choosing default value which is same as
8
+    cpu_count() used by ProcessPoolExecutor.

Loading…
Cancel
Save