stackforge/swiftsync
Code Issues Proposed changes

Rework the launching of the threads.

Browse Source 
- And make the max gthreads value configurable in ini.

Signed-off-by: Chmouel Boudjnah <chmouel@chmouel.com>
This commit is contained in:
Chmouel Boudjnah 2013-03-22 15:48:14 +01:00
parent 63de1098d1
commit a1f3df8727
2 changed files with 20 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
etc/config-sample.ini
View File

@ -7,6 +7,10 @@ keystone_dest = http://vm2:5000/v2.0
keystone_origin_demo_credentials = demo:demo:ADMIN keystone_origin_demo_credentials = demo:demo:ADMIN
keystone_dest_credentials = demo:demo:ADMIN keystone_dest_credentials = demo:demo:ADMIN
[sync]
# This is usually bound to the max open files.
max_gthreads = 500
[filler] [filler]
swift_operator_role = Member swift_operator_role = Member
concurrency = 1 concurrency = 1

26
sync/containers.py
View File

@ -2,6 +2,9 @@ import swiftclient
import eventlet import eventlet
from sync.objects import sync_object from sync.objects import sync_object
from common.utils import get_config
MAX_GTHREADS = get_config("sync", "max_gthreads")
def sync_container(orig_storage_cnx, orig_storage_url, def sync_container(orig_storage_cnx, orig_storage_url,
@ -38,18 +41,21 @@ def sync_container(orig_storage_cnx, orig_storage_url,
return return
pool = eventlet.GreenPool() pool = eventlet.GreenPool()
cnt = 0
count = 0
pile = eventlet.GreenPile(pool)
for obj in diff: for obj in diff:
pool.spawn_n(sync_object, pile.spawn(sync_object,
orig_storage_url, orig_storage_url,
orig_token, orig_token,
dest_storage_url, dest_storage_url,
dest_token, container_name, dest_token, container_name,
obj) obj)
if cnt == 20: if count == MAX_GTHREADS:
pool.waitall() pool.waitall()
cnt = 0 pile = eventlet.GreenPile(pool)
count = 0
else: else:
cnt += 1 count += 1
pool.waitall() pool.waitall()