Rework the launching of the threads.

- And make the max gthreads value configurable in ini.

Signed-off-by: Chmouel Boudjnah <chmouel@chmouel.com>

etc/config-sample.ini

@@ -7,6 +7,10 @@ keystone_dest = http://vm2:5000/v2.0
 keystone_origin_demo_credentials = demo:demo:ADMIN
 keystone_dest_credentials = demo:demo:ADMIN
 
+[sync]
+# This is usually  bound to the max open files.
+max_gthreads = 500
+
 [filler]
 swift_operator_role = Member
 concurrency = 1

sync/containers.py

@@ -2,6 +2,9 @@ import swiftclient
 import eventlet
 
 from sync.objects import sync_object
+from common.utils import get_config
+
+MAX_GTHREADS = get_config("sync", "max_gthreads")
 
 
 def sync_container(orig_storage_cnx, orig_storage_url,
@@ -38,18 +41,21 @@ def sync_container(orig_storage_cnx, orig_storage_url,
         return
 
     pool = eventlet.GreenPool()
-    cnt = 0
+
+    count = 0
+    pile = eventlet.GreenPile(pool)
     for obj in diff:
-        pool.spawn_n(sync_object,
-                     orig_storage_url,
-                     orig_token,
-                     dest_storage_url,
-                     dest_token, container_name,
-                     obj)
-        if cnt == 20:
+        pile.spawn(sync_object,
+                   orig_storage_url,
+                   orig_token,
+                   dest_storage_url,
+                   dest_token, container_name,
+                   obj)
+        if count == MAX_GTHREADS:
             pool.waitall()
-            cnt = 0
+            pile = eventlet.GreenPile(pool)
+            count = 0
         else:
-            cnt += 1
+            count += 1
 
     pool.waitall()